storagereplication stdout | 2025-11-04 08:54:28,138 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:54:28,531 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:28,781 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:28,936 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:29,048 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:29,073 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:29,127 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:54:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:54:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:44.351216+00:00 (in 14.999558 seconds) quotatotalworker stdout | 2025-11-04 08:54:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:54:44 GMT)" (scheduled at 2025-11-04 08:54:29.351216+00:00) quotatotalworker stdout | 2025-11-04 08:54:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:54:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:54:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:54:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:54:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:54:30,697 [287] [DEBUG] [app] Starting request: urn:request:91c20ece-29d0-41b2-b91e-422c3c24d141 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:54:30,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:30,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:54:30,701 [292] [DEBUG] [app] Starting request: urn:request:4f597bbc-d085-413f-a66d-43648f3d7e4f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:54:30,701 [292] [DEBUG] [app] Ending request: urn:request:4f597bbc-d085-413f-a66d-43648f3d7e4f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:4f597bbc-d085-413f-a66d-43648f3d7e4f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:54:30,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:54:30,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:30,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:30,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:54:30,705 [288] [DEBUG] [app] Starting request: urn:request:45ac6fc8-9161-4c23-92b9-0d954e62112b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:54:30,706 [288] [DEBUG] [app] Ending request: urn:request:45ac6fc8-9161-4c23-92b9-0d954e62112b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:45ac6fc8-9161-4c23-92b9-0d954e62112b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:54:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:54:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:30,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:54:30,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:54:30,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:54:30,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:54:30,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:54:30,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:54:30,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:54:30,730 [287] [DEBUG] [app] Ending request: urn:request:91c20ece-29d0-41b2-b91e-422c3c24d141 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:91c20ece-29d0-41b2-b91e-422c3c24d141', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:54:30,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:54:30,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:54:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:54:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 08:54:30,778 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:30,784 [288] [DEBUG] [app] Starting request: urn:request:fe54fadc-1c04-4a4e-9eec-67cc039d2cf5 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:54:30,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:30,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:54:30,788 [298] [DEBUG] [app] Starting request: urn:request:76e96b97-e1e4-4d02-bfd1-1abf6e2681eb (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:54:30,788 [298] [DEBUG] [app] Ending request: urn:request:76e96b97-e1e4-4d02-bfd1-1abf6e2681eb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:76e96b97-e1e4-4d02-bfd1-1abf6e2681eb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:54:30,788 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:54:30,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:30,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:30,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:54:30,792 [287] [DEBUG] [app] Starting request: urn:request:2c96001a-f531-4e2d-874e-fb12bc72c2a5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:54:30,792 [287] [DEBUG] [app] Ending request: urn:request:2c96001a-f531-4e2d-874e-fb12bc72c2a5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2c96001a-f531-4e2d-874e-fb12bc72c2a5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.000 159 0.001) gunicorn-web stdout | 2025-11-04 08:54:30,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:54:30,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:30,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:54:30,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:54:30,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:54:30,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:54:30,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:54:30,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:54:30,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:54:30,817 [288] [DEBUG] [app] Ending request: urn:request:fe54fadc-1c04-4a4e-9eec-67cc039d2cf5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fe54fadc-1c04-4a4e-9eec-67cc039d2cf5', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:54:30,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:54:30,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:54:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:54:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) notificationworker stdout | 2025-11-04 08:54:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:54:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:40.533243+00:00 (in 9.002775 seconds) notificationworker stdout | 2025-11-04 08:54:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:41 GMT)" (scheduled at 2025-11-04 08:54:31.530064+00:00) notificationworker stdout | 2025-11-04 08:54:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:54:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 31, 530738), True, datetime.datetime(2025, 11, 4, 8, 54, 31, 530738), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:54:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:54:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:54:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:41 GMT)" executed successfully gcworker stdout | 2025-11-04 08:54:31,571 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:54:31,770 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:54:31,774 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:54:32,072 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:54:32,085 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:54:32,085 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:54:32,095 [66] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:54:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:54:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:21.434460+00:00 (in 48.998190 seconds) namespacegcworker stdout | 2025-11-04 08:54:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:32 GMT)" (scheduled at 2025-11-04 08:54:32.435813+00:00) namespacegcworker stdout | 2025-11-04 08:54:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 08:54:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 08:54:33,055 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:54:33,068 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:54:33,171 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:54:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:54:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:48.625192+00:00 (in 15.001752 seconds) exportactionlogsworker stdout | 2025-11-04 08:54:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:33 GMT)" (scheduled at 2025-11-04 08:54:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 08:54:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 08:54:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 33, 623764), True, datetime.datetime(2025, 11, 4, 8, 54, 33, 623764), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 08:54:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 08:54:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 08:54:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:33 GMT)" executed successfully builder stdout | 2025-11-04 08:54:33,961 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:54:35,252 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:54:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:54:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:38.437104+00:00 (in 3.000889 seconds) gcworker stdout | 2025-11-04 08:54:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:05 GMT)" (scheduled at 2025-11-04 08:54:35.435809+00:00) gcworker stdout | 2025-11-04 08:54:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761036875436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:54:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:54:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:54:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:05 GMT)" executed successfully servicekey stdout | 2025-11-04 08:54:35,613 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:54:36,243 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:54:36,631 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:54:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:54:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:40.124445+00:00 (in 3.001150 seconds) proxycacheblobworker stdout | 2025-11-04 08:54:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:47 GMT)" (scheduled at 2025-11-04 08:54:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:54:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:54:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 37, 123595), True, datetime.datetime(2025, 11, 4, 8, 54, 37, 123595), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:54:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:54:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:54:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:54:37,351 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,362 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,402 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,413 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,414 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,444 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,495 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,585 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:54:37,615 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:54:37,903 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:54:37,906 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:54:37,926 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:54:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:54:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:05.435809+00:00 (in 26.998293 seconds) gcworker stdout | 2025-11-04 08:54:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:08 GMT)" (scheduled at 2025-11-04 08:54:38.437104+00:00) gcworker stdout | 2025-11-04 08:54:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:54:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246418449, None, 1, 0]) gcworker stdout | 2025-11-04 08:54:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:54:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:54:38,789 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:54:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:54:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:01.922827+00:00 (in 21.997803 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:54:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:39 GMT)" (scheduled at 2025-11-04 08:54:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:54:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 08:54:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 08:54:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:54:40,125 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:47.122849+00:00 (in 6.997691 seconds) proxycacheblobworker stdout | 2025-11-04 08:54:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:40 GMT)" (scheduled at 2025-11-04 08:54:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 08:54:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 08:54:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:54:40,527 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:54:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:54:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:41.530064+00:00 (in 0.996416 seconds) notificationworker stdout | 2025-11-04 08:54:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 08:59:40 GMT)" (scheduled at 2025-11-04 08:54:40.533243+00:00) notificationworker stdout | 2025-11-04 08:54:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 54, 40, 533862), 'notification/%']) notificationworker stdout | 2025-11-04 08:54:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 54, 40, 533862), True, datetime.datetime(2025, 11, 4, 8, 54, 40, 533862), 0, 'notification/%']) notificationworker stdout | 2025-11-04 08:54:40,550 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 54, 40, 533862), True, datetime.datetime(2025, 11, 4, 8, 54, 40, 533862), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 8, 54, 40, 533862), 'notification/%']) notificationworker stdout | 2025-11-04 08:54:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:54:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 08:59:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:54:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:54:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:51.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 08:54:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:51 GMT)" (scheduled at 2025-11-04 08:54:41.530064+00:00) notificationworker stdout | 2025-11-04 08:54:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:54:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 41, 530711), True, datetime.datetime(2025, 11, 4, 8, 54, 41, 530711), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:54:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:54:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:54:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:54:42,016 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:54:42,095 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:54:42,108 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:54:42,108 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:54:42,119 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:54:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:54:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:57.523915+00:00 (in 14.090638 seconds) repositorygcworker stdout | 2025-11-04 08:54:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:43 GMT)" (scheduled at 2025-11-04 08:54:43.432867+00:00) repositorygcworker stdout | 2025-11-04 08:54:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 08:54:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 43, 433620), True, datetime.datetime(2025, 11, 4, 8, 54, 43, 433620), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 08:54:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 08:54:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:54:43,447 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 08:54:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:54:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:59.351216+00:00 (in 14.999591 seconds) quotatotalworker stdout | 2025-11-04 08:54:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:54:59 GMT)" (scheduled at 2025-11-04 08:54:44.351216+00:00) quotatotalworker stdout | 2025-11-04 08:54:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:54:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:54:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:54:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:54:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:54:44,515 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:54:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:54:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:47.479080+00:00 (in 1.998909 seconds) securityworker stdout | 2025-11-04 08:54:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:15 GMT)" (scheduled at 2025-11-04 08:54:45.479769+00:00) securityworker stdout | 2025-11-04 08:54:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:54:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:54:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:54:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:54:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:54:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:54:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:54:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:54:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:54:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:54:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:54:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 49, 45, 483635), 1, 2]) securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:54:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:54:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 49, 45, 483635), 1, 2]) securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:54:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:54:45,697 [285] [DEBUG] [app] Starting request: urn:request:b21ddac7-8b2f-44f6-91dd-d56438e6af1c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:54:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:45,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:54:45,702 [292] [DEBUG] [app] Starting request: urn:request:d5793363-932b-4ecc-8f66-8ba34a4e4add (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:54:45,702 [292] [DEBUG] [app] Ending request: urn:request:d5793363-932b-4ecc-8f66-8ba34a4e4add (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d5793363-932b-4ecc-8f66-8ba34a4e4add', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:54:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 08:54:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:54:45,706 [288] [DEBUG] [app] Starting request: urn:request:80f4bfff-93e6-4f67-ad20-0bd476809fa2 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:54:45,706 [288] [DEBUG] [app] Ending request: urn:request:80f4bfff-93e6-4f67-ad20-0bd476809fa2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:80f4bfff-93e6-4f67-ad20-0bd476809fa2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:54:45,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:54:45,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:54:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:54:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:54:45,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:54:45,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:54:45,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:54:45,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:54:45,731 [285] [DEBUG] [app] Ending request: urn:request:b21ddac7-8b2f-44f6-91dd-d56438e6af1c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b21ddac7-8b2f-44f6-91dd-d56438e6af1c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:54:45,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:54:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:54:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:54:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:54:45,783 [288] [DEBUG] [app] Starting request: urn:request:fb0aa6ae-7dcb-4b51-80b0-1fbc97373a2e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:54:45,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:45,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:54:45,787 [292] [DEBUG] [app] Starting request: urn:request:6280dcd6-d54b-4e79-bfe7-c2c869dd60d3 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:54:45,788 [292] [DEBUG] [app] Ending request: urn:request:6280dcd6-d54b-4e79-bfe7-c2c869dd60d3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6280dcd6-d54b-4e79-bfe7-c2c869dd60d3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:54:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:54:45,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:45,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:54:45,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:54:45,792 [287] [DEBUG] [app] Starting request: urn:request:7acbf31f-1e49-457b-b366-983cdbb6fe0d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:54:45,792 [287] [DEBUG] [app] Ending request: urn:request:7acbf31f-1e49-457b-b366-983cdbb6fe0d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:7acbf31f-1e49-457b-b366-983cdbb6fe0d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:54:45,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:54:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:54:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:54:45,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:54:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:54:45,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:54:45,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:54:45,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:54:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:54:45,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:54:45,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:54:45,816 [288] [DEBUG] [app] Ending request: urn:request:fb0aa6ae-7dcb-4b51-80b0-1fbc97373a2e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fb0aa6ae-7dcb-4b51-80b0-1fbc97373a2e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:54:45,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:54:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:54:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:54:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) proxycacheblobworker stdout | 2025-11-04 08:54:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:54:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:53.124163+00:00 (in 6.000864 seconds) proxycacheblobworker stdout | 2025-11-04 08:54:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:57 GMT)" (scheduled at 2025-11-04 08:54:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:54:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:54:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 47, 123630), True, datetime.datetime(2025, 11, 4, 8, 54, 47, 123630), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:54:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:54:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:54:47,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:54:57 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:54:47,459 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:54:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:54:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:15.479769+00:00 (in 28.000232 seconds) securityworker stdout | 2025-11-04 08:54:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:17 GMT)" (scheduled at 2025-11-04 08:54:47.479080+00:00) securityworker stdout | 2025-11-04 08:54:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:54:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:54:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:54:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:54:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:54:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:54:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:54:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:54:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:54:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:54:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:54:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:54:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 49, 47, 483151), 1, 2]) securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:54:47,505 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:54:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:54:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 49, 47, 483151), 1, 2]) securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:54:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:54:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:54:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:54:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:54:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:54:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:54:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:54:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:54:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:17 GMT)" executed successfully reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:54:48,103 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:54:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:54:48,626 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:33.622985+00:00 (in 44.996839 seconds) exportactionlogsworker stdout | 2025-11-04 08:54:48,626 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:48 GMT)" (scheduled at 2025-11-04 08:54:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 08:54:48,626 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 08:54:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:54:49,422 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:54:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:54:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:21.226294+00:00 (in 29.999544 seconds) autopruneworker stdout | 2025-11-04 08:54:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:21 GMT)" (scheduled at 2025-11-04 08:54:51.226294+00:00) autopruneworker stdout | 2025-11-04 08:54:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246431235, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:54:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:54:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:54:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:54:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:54:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:01.530064+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 08:54:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:01 GMT)" (scheduled at 2025-11-04 08:54:51.530064+00:00) notificationworker stdout | 2025-11-04 08:54:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:54:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 51, 530713), True, datetime.datetime(2025, 11, 4, 8, 54, 51, 530713), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:54:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:54:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:54:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:54:51,697 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:54:52,119 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:54:52,132 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:54:52,132 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:54:52,143 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:54:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:54:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:54:57.122849+00:00 (in 3.998200 seconds) proxycacheblobworker stdout | 2025-11-04 08:54:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 08:59:53 GMT)" (scheduled at 2025-11-04 08:54:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 08:54:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 54, 53, 124922), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 08:54:53,136 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 54, 53, 124922), True, datetime.datetime(2025, 11, 4, 8, 54, 53, 124922), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 08:54:53,140 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 54, 53, 124922), True, datetime.datetime(2025, 11, 4, 8, 54, 53, 124922), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 8, 54, 53, 124922), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 08:54:53,143 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:54:53,143 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 08:59:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:54:54,533 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:54:56,787 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:54:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:54:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:07.122849+00:00 (in 9.999577 seconds) proxycacheblobworker stdout | 2025-11-04 08:54:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:07 GMT)" (scheduled at 2025-11-04 08:54:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:54:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:54:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 54, 57, 123571), True, datetime.datetime(2025, 11, 4, 8, 54, 57, 123571), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:54:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:54:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:09.133645+00:00 (in 12.001445 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:54:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:57 GMT)" (scheduled at 2025-11-04 08:54:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:54:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:54:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:54:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:54:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 08:54:57,144 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:54:57,144 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:54:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:54:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:54:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:27.222846+00:00 (in 29.999595 seconds) buildlogsarchiver stdout | 2025-11-04 08:54:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:27 GMT)" (scheduled at 2025-11-04 08:54:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:54:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 54, 57, 223512), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:54:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:54:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 08:54:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:54:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:54:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:27 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:54:57,502 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 08:54:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:54:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:43.432867+00:00 (in 45.908541 seconds) repositorygcworker stdout | 2025-11-04 08:54:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:57 GMT)" (scheduled at 2025-11-04 08:54:57.523915+00:00) repositorygcworker stdout | 2025-11-04 08:54:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 08:54:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:55:57 GMT)" executed successfully storagereplication stdout | 2025-11-04 08:54:58,148 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:54:58,541 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:58,790 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:58,947 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:59,062 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:59,086 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:54:59,141 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:54:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:54:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:14.351216+00:00 (in 14.999556 seconds) quotatotalworker stdout | 2025-11-04 08:54:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:14 GMT)" (scheduled at 2025-11-04 08:54:59.351216+00:00) quotatotalworker stdout | 2025-11-04 08:54:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:54:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:54:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:54:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:55:00,697 [285] [DEBUG] [app] Starting request: urn:request:26ded398-04c8-48c5-8d14-2ac7dcf472d9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:00,701 [292] [DEBUG] [app] Starting request: urn:request:efffb6ae-f192-4566-823f-5fcf4a353471 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:00,701 [292] [DEBUG] [app] Ending request: urn:request:efffb6ae-f192-4566-823f-5fcf4a353471 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:efffb6ae-f192-4566-823f-5fcf4a353471', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 08:55:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:00,705 [288] [DEBUG] [app] Starting request: urn:request:9ae9fcef-819e-41a0-86b8-c3bdda1a289a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:00,706 [288] [DEBUG] [app] Ending request: urn:request:9ae9fcef-819e-41a0-86b8-c3bdda1a289a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9ae9fcef-819e-41a0-86b8-c3bdda1a289a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:55:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 08:55:00,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:00,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:55:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:00,731 [285] [DEBUG] [app] Ending request: urn:request:26ded398-04c8-48c5-8d14-2ac7dcf472d9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:26ded398-04c8-48c5-8d14-2ac7dcf472d9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:55:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 08:55:00,784 [287] [DEBUG] [app] Starting request: urn:request:b1882272-5ca3-4b8f-b653-a879bb8b6a09 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:00,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:00,788 [297] [DEBUG] [app] Starting request: urn:request:a327aadc-c098-4d4d-b12a-e94b4908743f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:00,788 [297] [DEBUG] [app] Ending request: urn:request:a327aadc-c098-4d4d-b12a-e94b4908743f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a327aadc-c098-4d4d-b12a-e94b4908743f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} manifestsubjectbackfillworker stdout | 2025-11-04 08:55:00,788 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 08:55:00,789 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:00,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:00,790 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:00,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:00,792 [286] [DEBUG] [app] Starting request: urn:request:cade35fc-3342-4e67-9d10-6358eacdc23e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:00,793 [286] [DEBUG] [app] Ending request: urn:request:cade35fc-3342-4e67-9d10-6358eacdc23e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cade35fc-3342-4e67-9d10-6358eacdc23e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:55:00,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:00,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:00,794 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:00,794 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:55:00,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:00,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:00,810 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:00,814 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:00,817 [287] [DEBUG] [app] Ending request: urn:request:b1882272-5ca3-4b8f-b653-a879bb8b6a09 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b1882272-5ca3-4b8f-b653-a879bb8b6a09', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:00,817 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:55:00,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) notificationworker stdout | 2025-11-04 08:55:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:55:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:11.530064+00:00 (in 9.999519 seconds) notificationworker stdout | 2025-11-04 08:55:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:11 GMT)" (scheduled at 2025-11-04 08:55:01.530064+00:00) notificationworker stdout | 2025-11-04 08:55:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:55:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 1, 530814), True, datetime.datetime(2025, 11, 4, 8, 55, 1, 530814), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:55:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:55:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:11 GMT)" executed successfully gcworker stdout | 2025-11-04 08:55:01,580 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:55:01,779 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:55:01,783 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:55:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:55:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:39.924652+00:00 (in 38.001422 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:55:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:01 GMT)" (scheduled at 2025-11-04 08:55:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:55:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 08:55:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 1, 923504), True, datetime.datetime(2025, 11, 4, 8, 55, 1, 923504), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 08:55:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 08:55:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 08:55:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:01 GMT)" executed successfully builder stdout | 2025-11-04 08:55:02,143 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:55:02,156 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:55:02,156 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:55:02,167 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:55:03,064 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:55:03,076 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:55:03,181 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:55:03,970 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:55:05,261 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:55:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:55:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:08.437104+00:00 (in 3.000904 seconds) gcworker stdout | 2025-11-04 08:55:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:35 GMT)" (scheduled at 2025-11-04 08:55:05.435809+00:00) gcworker stdout | 2025-11-04 08:55:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761036905436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:55:05,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:55:05,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:55:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:35 GMT)" executed successfully servicekey stdout | 2025-11-04 08:55:05,622 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:55:06,252 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:55:06,640 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:55:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:55:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:17.122849+00:00 (in 9.999551 seconds) proxycacheblobworker stdout | 2025-11-04 08:55:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:17 GMT)" (scheduled at 2025-11-04 08:55:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:55:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:55:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 7, 123610), True, datetime.datetime(2025, 11, 4, 8, 55, 7, 123610), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:55:07,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:55:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:55:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:55:07,365 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,375 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,416 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,424 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,425 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,457 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,505 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,600 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:07,630 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:55:07,914 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:55:07,917 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:55:07,936 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:55:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:55:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:35.435809+00:00 (in 26.998238 seconds) gcworker stdout | 2025-11-04 08:55:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:38 GMT)" (scheduled at 2025-11-04 08:55:08.437104+00:00) gcworker stdout | 2025-11-04 08:55:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:55:08,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246448450, None, 1, 0]) gcworker stdout | 2025-11-04 08:55:08,454 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:55:08,454 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:55:08,798 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 08:55:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:55:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:57.131795+00:00 (in 47.997744 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:55:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:09 GMT)" (scheduled at 2025-11-04 08:55:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:55:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:55:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:55:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:55:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:55:10,536 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:55:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:55:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:15.535095+00:00 (in 4.004560 seconds) notificationworker stdout | 2025-11-04 08:55:11,531 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:21 GMT)" (scheduled at 2025-11-04 08:55:11.530064+00:00) notificationworker stdout | 2025-11-04 08:55:11,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:55:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 11, 531280), True, datetime.datetime(2025, 11, 4, 8, 55, 11, 531280), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:55:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:55:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:55:12,024 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:55:12,167 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:55:12,181 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:55:12,181 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:55:12,191 [66] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 08:55:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 08:55:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:12.222828+00:00 (in 59.999580 seconds) quotaregistrysizeworker stdout | 2025-11-04 08:55:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:12 GMT)" (scheduled at 2025-11-04 08:55:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 08:55:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 08:55:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 08:55:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:12 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 08:55:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:55:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:29.351216+00:00 (in 14.999556 seconds) quotatotalworker stdout | 2025-11-04 08:55:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:29 GMT)" (scheduled at 2025-11-04 08:55:14.351216+00:00) quotatotalworker stdout | 2025-11-04 08:55:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:55:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:55:14,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:55:14,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:55:14,524 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:55:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:55:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:17.479080+00:00 (in 1.998884 seconds) securityworker stdout | 2025-11-04 08:55:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:45 GMT)" (scheduled at 2025-11-04 08:55:15.479769+00:00) securityworker stdout | 2025-11-04 08:55:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:55:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:55:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:55:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 50, 15, 484104), 1, 2]) securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 50, 15, 484104), 1, 2]) securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:55:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:55:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:21.530064+00:00 (in 5.994464 seconds) notificationworker stdout | 2025-11-04 08:55:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:15 GMT)" (scheduled at 2025-11-04 08:55:15.535095+00:00) notificationworker stdout | 2025-11-04 08:55:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 08:55:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:55:15,697 [285] [DEBUG] [app] Starting request: urn:request:62f3469d-a0f5-445c-83a3-f84be90037ea (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:15,701 [292] [DEBUG] [app] Starting request: urn:request:0a923ad0-269b-4038-9a64-694c8c7494e2 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:15,702 [292] [DEBUG] [app] Ending request: urn:request:0a923ad0-269b-4038-9a64-694c8c7494e2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0a923ad0-269b-4038-9a64-694c8c7494e2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 08:55:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:15,706 [288] [DEBUG] [app] Starting request: urn:request:672f70e1-5844-4f68-97c8-960509db0fce (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:15,706 [288] [DEBUG] [app] Ending request: urn:request:672f70e1-5844-4f68-97c8-960509db0fce (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:672f70e1-5844-4f68-97c8-960509db0fce', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:55:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:55:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:55:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:15,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:15,730 [285] [DEBUG] [app] Ending request: urn:request:62f3469d-a0f5-445c-83a3-f84be90037ea (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:62f3469d-a0f5-445c-83a3-f84be90037ea', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:15,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:55:15,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:55:15,784 [285] [DEBUG] [app] Starting request: urn:request:3afc52f3-9fa8-49fc-b5ed-27104f9689cd (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:15,785 [292] [DEBUG] [app] Starting request: urn:request:142b3301-d6ed-4997-9318-a8e19a2a56cb (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:15,785 [292] [DEBUG] [app] Ending request: urn:request:142b3301-d6ed-4997-9318-a8e19a2a56cb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:142b3301-d6ed-4997-9318-a8e19a2a56cb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:55:15,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:55:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:15,789 [288] [DEBUG] [app] Starting request: urn:request:e67b71c8-d1bc-4f1b-93d3-43615315d104 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:15,790 [288] [DEBUG] [app] Ending request: urn:request:e67b71c8-d1bc-4f1b-93d3-43615315d104 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e67b71c8-d1bc-4f1b-93d3-43615315d104', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:55:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:55:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:55:15,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:15,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:15,813 [285] [DEBUG] [app] Ending request: urn:request:3afc52f3-9fa8-49fc-b5ed-27104f9689cd (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3afc52f3-9fa8-49fc-b5ed-27104f9689cd', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:15,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:55:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) proxycacheblobworker stdout | 2025-11-04 08:55:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:55:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:27.122849+00:00 (in 9.999636 seconds) proxycacheblobworker stdout | 2025-11-04 08:55:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:27 GMT)" (scheduled at 2025-11-04 08:55:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:55:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:55:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 17, 123477), True, datetime.datetime(2025, 11, 4, 8, 55, 17, 123477), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:55:17,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:55:17,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:55:17,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:27 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:55:17,468 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:55:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:55:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:45.479769+00:00 (in 28.000305 seconds) securityworker stdout | 2025-11-04 08:55:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:47 GMT)" (scheduled at 2025-11-04 08:55:17.479080+00:00) securityworker stdout | 2025-11-04 08:55:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:55:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:55:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:55:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:55:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:55:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:47 GMT)" executed successfully reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:55:18,112 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 08:55:19,431 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:55:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:55:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:51.226294+00:00 (in 29.999553 seconds) autopruneworker stdout | 2025-11-04 08:55:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:51 GMT)" (scheduled at 2025-11-04 08:55:21.226294+00:00) autopruneworker stdout | 2025-11-04 08:55:21,234 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246461234, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:55:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:55:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:55:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 08:55:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:55:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:32.435813+00:00 (in 11.000925 seconds) namespacegcworker stdout | 2025-11-04 08:55:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:21 GMT)" (scheduled at 2025-11-04 08:55:21.434460+00:00) namespacegcworker stdout | 2025-11-04 08:55:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 08:55:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 21, 435170), True, datetime.datetime(2025, 11, 4, 8, 55, 21, 435170), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 08:55:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 08:55:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:55:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:55:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:55:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:31.530064+00:00 (in 9.999561 seconds) notificationworker stdout | 2025-11-04 08:55:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:31 GMT)" (scheduled at 2025-11-04 08:55:21.530064+00:00) notificationworker stdout | 2025-11-04 08:55:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:55:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 21, 530748), True, datetime.datetime(2025, 11, 4, 8, 55, 21, 530748), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:55:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:55:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:55:21,706 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:55:22,191 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:55:22,206 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:55:22,206 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:55:22,216 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:24,543 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:55:26,796 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:55:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:55:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:37.122849+00:00 (in 9.999642 seconds) proxycacheblobworker stdout | 2025-11-04 08:55:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:37 GMT)" (scheduled at 2025-11-04 08:55:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:55:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:55:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 27, 123455), True, datetime.datetime(2025, 11, 4, 8, 55, 27, 123455), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:55:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:55:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:55:27,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:55:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:55:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:57.222846+00:00 (in 29.999590 seconds) buildlogsarchiver stdout | 2025-11-04 08:55:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:57 GMT)" (scheduled at 2025-11-04 08:55:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:55:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 55, 27, 223461), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:55:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:55:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 08:55:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:55:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:55:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:55:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:55:27,511 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:55:28,157 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:55:28,550 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:28,800 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:28,958 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:29,076 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:29,100 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:29,155 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:55:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:55:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:44.351216+00:00 (in 14.999618 seconds) quotatotalworker stdout | 2025-11-04 08:55:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:44 GMT)" (scheduled at 2025-11-04 08:55:29.351216+00:00) quotatotalworker stdout | 2025-11-04 08:55:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:55:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:55:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:55:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:55:30,696 [287] [DEBUG] [app] Starting request: urn:request:649a6481-d1ac-4749-8c6b-7bc382b46796 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:30,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:30,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:30,701 [299] [DEBUG] [app] Starting request: urn:request:db7db6f4-9b40-4c41-b626-2f4dcca41e85 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:30,701 [299] [DEBUG] [app] Ending request: urn:request:db7db6f4-9b40-4c41-b626-2f4dcca41e85 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:db7db6f4-9b40-4c41-b626-2f4dcca41e85', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 08:55:30,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:30,701 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:30,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:30,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:30,705 [288] [DEBUG] [app] Starting request: urn:request:e6bdd01b-4d63-4784-935b-9bb6cf11adba (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:30,705 [288] [DEBUG] [app] Ending request: urn:request:e6bdd01b-4d63-4784-935b-9bb6cf11adba (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e6bdd01b-4d63-4784-935b-9bb6cf11adba', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 08:55:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:30,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:30,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:30,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:55:30,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:30,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:30,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:30,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:30,730 [287] [DEBUG] [app] Ending request: urn:request:649a6481-d1ac-4749-8c6b-7bc382b46796 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:649a6481-d1ac-4749-8c6b-7bc382b46796', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:30,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:55:30,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:55:30,783 [285] [DEBUG] [app] Starting request: urn:request:1d0ee48e-1912-45f0-8b8f-c08c4c59d1b3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:30,788 [293] [DEBUG] [app] Starting request: urn:request:5cf3d0a7-d3fb-4011-9609-507ca56114bf (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:30,788 [293] [DEBUG] [app] Ending request: urn:request:5cf3d0a7-d3fb-4011-9609-507ca56114bf (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5cf3d0a7-d3fb-4011-9609-507ca56114bf', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 08:55:30,788 [293] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:30,792 [288] [DEBUG] [app] Starting request: urn:request:dfeaa382-0662-42a0-9718-674a5370d9ed (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:30,792 [288] [DEBUG] [app] Ending request: urn:request:dfeaa382-0662-42a0-9718-674a5370d9ed (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dfeaa382-0662-42a0-9718-674a5370d9ed', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:55:30,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:55:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql manifestsubjectbackfillworker stdout | 2025-11-04 08:55:30,798 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:30,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:30,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:30,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:30,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:30,817 [285] [DEBUG] [app] Ending request: urn:request:1d0ee48e-1912-45f0-8b8f-c08c4c59d1b3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1d0ee48e-1912-45f0-8b8f-c08c4c59d1b3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:30,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:55:30,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" notificationworker stdout | 2025-11-04 08:55:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:55:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:41.530064+00:00 (in 9.999530 seconds) notificationworker stdout | 2025-11-04 08:55:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:41 GMT)" (scheduled at 2025-11-04 08:55:31.530064+00:00) notificationworker stdout | 2025-11-04 08:55:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:55:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 31, 530818), True, datetime.datetime(2025, 11, 4, 8, 55, 31, 530818), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:55:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:55:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:41 GMT)" executed successfully gcworker stdout | 2025-11-04 08:55:31,588 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:55:31,788 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:55:31,792 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:55:32,216 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:55:32,231 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:55:32,231 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:55:32,241 [66] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:55:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:55:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:21.434460+00:00 (in 48.998229 seconds) namespacegcworker stdout | 2025-11-04 08:55:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:32 GMT)" (scheduled at 2025-11-04 08:55:32.435813+00:00) namespacegcworker stdout | 2025-11-04 08:55:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 08:55:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 08:55:33,073 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:55:33,085 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:55:33,190 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:55:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:55:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:48.625192+00:00 (in 15.001778 seconds) exportactionlogsworker stdout | 2025-11-04 08:55:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:33 GMT)" (scheduled at 2025-11-04 08:55:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 08:55:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 08:55:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 33, 623717), True, datetime.datetime(2025, 11, 4, 8, 55, 33, 623717), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 08:55:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 08:55:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 08:55:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:33 GMT)" executed successfully builder stdout | 2025-11-04 08:55:33,980 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:55:35,270 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:55:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:55:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:38.437104+00:00 (in 3.000911 seconds) gcworker stdout | 2025-11-04 08:55:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:05 GMT)" (scheduled at 2025-11-04 08:55:35.435809+00:00) gcworker stdout | 2025-11-04 08:55:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761036935436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:55:35,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:55:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:55:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:05 GMT)" executed successfully servicekey stdout | 2025-11-04 08:55:35,631 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:55:36,262 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:55:36,649 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:55:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:55:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:40.124445+00:00 (in 3.001179 seconds) proxycacheblobworker stdout | 2025-11-04 08:55:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:47 GMT)" (scheduled at 2025-11-04 08:55:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:55:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:55:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 37, 123562), True, datetime.datetime(2025, 11, 4, 8, 55, 37, 123562), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:55:37,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:55:37,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:55:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:55:37,378 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,389 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,431 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,435 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,435 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,468 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,518 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,613 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:55:37,645 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:55:37,925 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:55:37,928 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:55:37,946 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:55:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:55:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:05.435809+00:00 (in 26.998083 seconds) gcworker stdout | 2025-11-04 08:55:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:08 GMT)" (scheduled at 2025-11-04 08:55:38.437104+00:00) gcworker stdout | 2025-11-04 08:55:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:55:38,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246478449, None, 1, 0]) gcworker stdout | 2025-11-04 08:55:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:55:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:55:38,807 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:55:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:55:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:01.922827+00:00 (in 21.997784 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:55:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:39 GMT)" (scheduled at 2025-11-04 08:55:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:55:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 08:55:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 08:55:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:55:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:47.122849+00:00 (in 6.998012 seconds) proxycacheblobworker stdout | 2025-11-04 08:55:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:40 GMT)" (scheduled at 2025-11-04 08:55:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 08:55:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 08:55:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:55:40,544 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:55:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:55:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:51.530064+00:00 (in 9.999513 seconds) notificationworker stdout | 2025-11-04 08:55:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:51 GMT)" (scheduled at 2025-11-04 08:55:41.530064+00:00) notificationworker stdout | 2025-11-04 08:55:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:55:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 41, 530837), True, datetime.datetime(2025, 11, 4, 8, 55, 41, 530837), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:55:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:55:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:55:42,033 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:55:42,242 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:55:42,255 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:55:42,255 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:55:42,266 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:55:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:55:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:57.523915+00:00 (in 14.090665 seconds) repositorygcworker stdout | 2025-11-04 08:55:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:43 GMT)" (scheduled at 2025-11-04 08:55:43.432867+00:00) repositorygcworker stdout | 2025-11-04 08:55:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 08:55:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 43, 433590), True, datetime.datetime(2025, 11, 4, 8, 55, 43, 433590), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 08:55:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 08:55:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:55:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 08:55:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:55:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:59.351216+00:00 (in 14.999575 seconds) quotatotalworker stdout | 2025-11-04 08:55:44,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:59 GMT)" (scheduled at 2025-11-04 08:55:44.351216+00:00) quotatotalworker stdout | 2025-11-04 08:55:44,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:55:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:55:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:55:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:55:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:55:44,532 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:55:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:55:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:47.479080+00:00 (in 1.998895 seconds) securityworker stdout | 2025-11-04 08:55:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:15 GMT)" (scheduled at 2025-11-04 08:55:45.479769+00:00) securityworker stdout | 2025-11-04 08:55:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:55:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:55:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:55:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:55:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 50, 45, 483468), 1, 2]) securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 50, 45, 483468), 1, 2]) securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:55:45,696 [287] [DEBUG] [app] Starting request: urn:request:577410a1-1855-4f35-a368-cfdf369e91b5 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:45,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:45,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:45,701 [298] [DEBUG] [app] Starting request: urn:request:0dfc10f9-e9c0-4503-889e-fb2a257a3004 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:45,701 [298] [DEBUG] [app] Ending request: urn:request:0dfc10f9-e9c0-4503-889e-fb2a257a3004 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0dfc10f9-e9c0-4503-889e-fb2a257a3004', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:55:45,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:55:45,701 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:45,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:45,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:45,705 [285] [DEBUG] [app] Starting request: urn:request:cc4638da-b0c4-4d66-a12a-e8347a415875 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:45,705 [285] [DEBUG] [app] Ending request: urn:request:cc4638da-b0c4-4d66-a12a-e8347a415875 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cc4638da-b0c4-4d66-a12a-e8347a415875', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:55:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 08:55:45,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:45,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:45,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:45,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:55:45,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:45,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:45,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:45,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:45,730 [287] [DEBUG] [app] Ending request: urn:request:577410a1-1855-4f35-a368-cfdf369e91b5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:577410a1-1855-4f35-a368-cfdf369e91b5', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:45,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:55:45,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:55:45,783 [285] [DEBUG] [app] Starting request: urn:request:bb351e02-8afa-4da3-9560-4bef66de9567 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:55:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:55:45,787 [298] [DEBUG] [app] Starting request: urn:request:174d29f9-1055-41f8-b30b-b0e8194f8913 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:55:45,788 [298] [DEBUG] [app] Ending request: urn:request:174d29f9-1055-41f8-b30b-b0e8194f8913 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:174d29f9-1055-41f8-b30b-b0e8194f8913', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:55:45,788 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:55:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:55:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:55:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:55:45,792 [288] [DEBUG] [app] Starting request: urn:request:b6c09dc7-cb85-4224-ada1-248245dc74d2 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:55:45,792 [288] [DEBUG] [app] Ending request: urn:request:b6c09dc7-cb85-4224-ada1-248245dc74d2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b6c09dc7-cb85-4224-ada1-248245dc74d2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:55:45,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:55:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:55:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:55:45,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:55:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:55:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:55:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:55:45,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:55:45,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:55:45,809 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:55:45,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:55:45,815 [285] [DEBUG] [app] Ending request: urn:request:bb351e02-8afa-4da3-9560-4bef66de9567 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bb351e02-8afa-4da3-9560-4bef66de9567', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:55:45,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:55:45,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:55:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:55:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.033) proxycacheblobworker stdout | 2025-11-04 08:55:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:55:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:55:57.122849+00:00 (in 9.999559 seconds) proxycacheblobworker stdout | 2025-11-04 08:55:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:57 GMT)" (scheduled at 2025-11-04 08:55:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:55:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:55:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 47, 123563), True, datetime.datetime(2025, 11, 4, 8, 55, 47, 123563), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:55:47,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:55:47,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:55:47,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:55:57 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:55:47,477 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:55:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:55:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:15.479769+00:00 (in 28.000269 seconds) securityworker stdout | 2025-11-04 08:55:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:17 GMT)" (scheduled at 2025-11-04 08:55:47.479080+00:00) securityworker stdout | 2025-11-04 08:55:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:55:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:55:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:55:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:55:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:47,497 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:47,501 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 50, 47, 483055), 1, 2]) securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:55:47,505 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:55:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 50, 47, 483055), 1, 2]) securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:47,509 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:55:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:55:47,509 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:17 GMT)" executed successfully reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:55:48,121 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:55:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:55:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:33.622985+00:00 (in 44.997281 seconds) exportactionlogsworker stdout | 2025-11-04 08:55:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:48 GMT)" (scheduled at 2025-11-04 08:55:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 08:55:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 08:55:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:55:49,440 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:55:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:55:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:21.226294+00:00 (in 29.999602 seconds) autopruneworker stdout | 2025-11-04 08:55:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:21 GMT)" (scheduled at 2025-11-04 08:55:51.226294+00:00) autopruneworker stdout | 2025-11-04 08:55:51,234 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246491234, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:55:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:55:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:55:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:55:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:55:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:01.530064+00:00 (in 9.999548 seconds) notificationworker stdout | 2025-11-04 08:55:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:01 GMT)" (scheduled at 2025-11-04 08:55:51.530064+00:00) notificationworker stdout | 2025-11-04 08:55:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:55:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 51, 530799), True, datetime.datetime(2025, 11, 4, 8, 55, 51, 530799), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:55:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:55:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:55:51,715 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:55:52,266 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:55:52,279 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:55:52,279 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:55:52,290 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:55:54,563 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:55:56,804 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:55:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:55:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:07.122849+00:00 (in 9.999604 seconds) proxycacheblobworker stdout | 2025-11-04 08:55:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:07 GMT)" (scheduled at 2025-11-04 08:55:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:55:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:55:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 55, 57, 123521), True, datetime.datetime(2025, 11, 4, 8, 55, 57, 123521), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:55:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:55:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:09.133645+00:00 (in 12.001462 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:55:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:57 GMT)" (scheduled at 2025-11-04 08:55:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:55:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:55:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:55:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:55:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 08:55:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:55:57,144 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:55:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:55:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:55:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:27.222846+00:00 (in 29.999592 seconds) buildlogsarchiver stdout | 2025-11-04 08:55:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:27 GMT)" (scheduled at 2025-11-04 08:55:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:55:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 55, 57, 223474), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:55:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:55:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 08:55:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:55:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:55:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:27 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:55:57,520 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 08:55:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:55:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:43.432867+00:00 (in 45.908540 seconds) repositorygcworker stdout | 2025-11-04 08:55:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:57 GMT)" (scheduled at 2025-11-04 08:55:57.523915+00:00) repositorygcworker stdout | 2025-11-04 08:55:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 08:55:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:56:57 GMT)" executed successfully storagereplication stdout | 2025-11-04 08:55:58,167 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:55:58,558 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:58,822 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:58,969 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:59,090 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:59,114 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:55:59,169 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:55:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:55:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:14.351216+00:00 (in 14.999575 seconds) quotatotalworker stdout | 2025-11-04 08:55:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:14 GMT)" (scheduled at 2025-11-04 08:55:59.351216+00:00) quotatotalworker stdout | 2025-11-04 08:55:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:55:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:55:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:55:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:56:00,696 [285] [DEBUG] [app] Starting request: urn:request:d7bf61a6-9200-483d-982c-f865cb0a17cd (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:00,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:00,701 [292] [DEBUG] [app] Starting request: urn:request:6c0b4276-fa01-4903-895b-d25e8bae25b7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:00,701 [292] [DEBUG] [app] Ending request: urn:request:6c0b4276-fa01-4903-895b-d25e8bae25b7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6c0b4276-fa01-4903-895b-d25e8bae25b7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:56:00,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 08:56:00,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:00,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:00,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:00,705 [288] [DEBUG] [app] Starting request: urn:request:dd49f93c-ef42-4d6e-87ca-ba6c40736f0c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:00,705 [288] [DEBUG] [app] Ending request: urn:request:dd49f93c-ef42-4d6e-87ca-ba6c40736f0c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dd49f93c-ef42-4d6e-87ca-ba6c40736f0c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:56:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 08:56:00,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:00,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:00,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:00,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:56:00,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:56:00,730 [285] [DEBUG] [app] Ending request: urn:request:d7bf61a6-9200-483d-982c-f865cb0a17cd (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d7bf61a6-9200-483d-982c-f865cb0a17cd', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:00,730 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 08:56:00,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 08:56:00,783 [285] [DEBUG] [app] Starting request: urn:request:0ae5ecc1-1e70-4efe-8975-9e2a7974bbf8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:00,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:00,785 [299] [DEBUG] [app] Starting request: urn:request:80cccb38-aba1-4559-906d-713182ecae2d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:00,785 [299] [DEBUG] [app] Ending request: urn:request:80cccb38-aba1-4559-906d-713182ecae2d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:80cccb38-aba1-4559-906d-713182ecae2d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 08:56:00,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:56:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:00,789 [287] [DEBUG] [app] Starting request: urn:request:6f930178-ffb5-4f57-bc64-22deb8b19231 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:00,790 [287] [DEBUG] [app] Ending request: urn:request:6f930178-ffb5-4f57-bc64-22deb8b19231 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6f930178-ffb5-4f57-bc64-22deb8b19231', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:56:00,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:56:00,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:00,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:00,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:00,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:00,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:00,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:00,807 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:00,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:56:00,814 [285] [DEBUG] [app] Ending request: urn:request:0ae5ecc1-1e70-4efe-8975-9e2a7974bbf8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0ae5ecc1-1e70-4efe-8975-9e2a7974bbf8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:00,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:56:00,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) notificationworker stdout | 2025-11-04 08:56:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:56:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:11.530064+00:00 (in 9.999528 seconds) notificationworker stdout | 2025-11-04 08:56:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:11 GMT)" (scheduled at 2025-11-04 08:56:01.530064+00:00) notificationworker stdout | 2025-11-04 08:56:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:56:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 1, 530798), True, datetime.datetime(2025, 11, 4, 8, 56, 1, 530798), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:56:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:56:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:11 GMT)" executed successfully gcworker stdout | 2025-11-04 08:56:01,608 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:56:01,796 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:56:01,812 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:56:01,922 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:56:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:39.924652+00:00 (in 38.001456 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:56:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:01 GMT)" (scheduled at 2025-11-04 08:56:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:56:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 08:56:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 1, 923442), True, datetime.datetime(2025, 11, 4, 8, 56, 1, 923442), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 08:56:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 08:56:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 08:56:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:01 GMT)" executed successfully builder stdout | 2025-11-04 08:56:02,290 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:56:02,303 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:56:02,303 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:56:02,313 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:56:03,082 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:56:03,094 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:56:03,199 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:56:03,989 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:56:05,279 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:56:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:56:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:08.437104+00:00 (in 3.000907 seconds) gcworker stdout | 2025-11-04 08:56:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:35 GMT)" (scheduled at 2025-11-04 08:56:05.435809+00:00) gcworker stdout | 2025-11-04 08:56:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761036965436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:56:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:56:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:56:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:35 GMT)" executed successfully servicekey stdout | 2025-11-04 08:56:05,640 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:56:06,271 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:56:06,658 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:56:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:56:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:17.122849+00:00 (in 9.999576 seconds) proxycacheblobworker stdout | 2025-11-04 08:56:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:17 GMT)" (scheduled at 2025-11-04 08:56:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:56:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:56:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 7, 123545), True, datetime.datetime(2025, 11, 4, 8, 56, 7, 123545), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:56:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:56:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:56:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:56:07,390 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,433 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,453 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,455 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,460 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,479 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,531 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,634 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:07,665 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:56:07,936 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:56:07,939 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:56:07,956 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:56:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:56:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:35.435809+00:00 (in 26.998302 seconds) gcworker stdout | 2025-11-04 08:56:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:38 GMT)" (scheduled at 2025-11-04 08:56:08.437104+00:00) gcworker stdout | 2025-11-04 08:56:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:56:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246508449, None, 1, 0]) gcworker stdout | 2025-11-04 08:56:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:56:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:56:08,817 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 08:56:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:56:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:57.131795+00:00 (in 47.997709 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:09 GMT)" (scheduled at 2025-11-04 08:56:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:56:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:56:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:56:10,553 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:56:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:56:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:15.535095+00:00 (in 4.004584 seconds) notificationworker stdout | 2025-11-04 08:56:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:21 GMT)" (scheduled at 2025-11-04 08:56:11.530064+00:00) notificationworker stdout | 2025-11-04 08:56:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:56:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 11, 530752), True, datetime.datetime(2025, 11, 4, 8, 56, 11, 530752), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:56:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:56:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:56:12,045 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:56:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 08:56:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:12.222828+00:00 (in 59.999558 seconds) quotaregistrysizeworker stdout | 2025-11-04 08:56:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:12 GMT)" (scheduled at 2025-11-04 08:56:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 08:56:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 08:56:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 08:56:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:12 GMT)" executed successfully builder stdout | 2025-11-04 08:56:12,313 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:56:12,327 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:56:12,327 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:56:12,336 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:56:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:56:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:29.351216+00:00 (in 14.999576 seconds) quotatotalworker stdout | 2025-11-04 08:56:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:29 GMT)" (scheduled at 2025-11-04 08:56:14.351216+00:00) quotatotalworker stdout | 2025-11-04 08:56:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:56:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:56:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:56:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:56:14,541 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:56:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:56:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:17.479080+00:00 (in 1.998891 seconds) securityworker stdout | 2025-11-04 08:56:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:45 GMT)" (scheduled at 2025-11-04 08:56:15.479769+00:00) securityworker stdout | 2025-11-04 08:56:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:56:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:56:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:56:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:56:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 51, 15, 483864), 1, 2]) securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 51, 15, 483864), 1, 2]) securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:56:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:56:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:56:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:56:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:21.530064+00:00 (in 5.994518 seconds) notificationworker stdout | 2025-11-04 08:56:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:15 GMT)" (scheduled at 2025-11-04 08:56:15.535095+00:00) notificationworker stdout | 2025-11-04 08:56:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 08:56:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:56:15,697 [286] [DEBUG] [app] Starting request: urn:request:9df3c96c-0601-4a01-9ff1-c65090447b6a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:15,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:15,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:15,702 [292] [DEBUG] [app] Starting request: urn:request:9f708703-0bf5-4da6-9c4c-fa5e9acd476c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:15,702 [292] [DEBUG] [app] Ending request: urn:request:9f708703-0bf5-4da6-9c4c-fa5e9acd476c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9f708703-0bf5-4da6-9c4c-fa5e9acd476c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:56:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 08:56:15,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:15,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:15,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:15,706 [285] [DEBUG] [app] Starting request: urn:request:1560759a-7d76-4ded-b5c6-41c103b3ada9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:15,706 [285] [DEBUG] [app] Ending request: urn:request:1560759a-7d76-4ded-b5c6-41c103b3ada9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1560759a-7d76-4ded-b5c6-41c103b3ada9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:56:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:56:15,707 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:15,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:15,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:15,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:15,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:15,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:15,723 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:56:15,726 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:56:15,729 [286] [DEBUG] [app] Ending request: urn:request:9df3c96c-0601-4a01-9ff1-c65090447b6a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9df3c96c-0601-4a01-9ff1-c65090447b6a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:15,730 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:56:15,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) gunicorn-web stdout | 2025-11-04 08:56:15,784 [285] [DEBUG] [app] Starting request: urn:request:ce580d62-531d-4415-8b9a-992ec042a1db (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:15,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:15,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:15,788 [292] [DEBUG] [app] Starting request: urn:request:41ab8b77-b96c-4b94-9972-7d0d92c62db0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:15,789 [292] [DEBUG] [app] Ending request: urn:request:41ab8b77-b96c-4b94-9972-7d0d92c62db0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:41ab8b77-b96c-4b94-9972-7d0d92c62db0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:56:15,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:56:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:15,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:15,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:15,793 [288] [DEBUG] [app] Starting request: urn:request:bc280fd9-85a0-4248-b727-24eb6bf5edd0 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:15,793 [288] [DEBUG] [app] Ending request: urn:request:bc280fd9-85a0-4248-b727-24eb6bf5edd0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bc280fd9-85a0-4248-b727-24eb6bf5edd0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:56:15,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:56:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:15,794 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:15,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:15,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:56:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:56:15,817 [285] [DEBUG] [app] Ending request: urn:request:ce580d62-531d-4415-8b9a-992ec042a1db (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ce580d62-531d-4415-8b9a-992ec042a1db', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:56:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 08:56:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:56:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:27.122849+00:00 (in 9.999595 seconds) proxycacheblobworker stdout | 2025-11-04 08:56:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:27 GMT)" (scheduled at 2025-11-04 08:56:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:56:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:56:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 17, 123562), True, datetime.datetime(2025, 11, 4, 8, 56, 17, 123562), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:56:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:56:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:56:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:27 GMT)" executed successfully securityworker stdout | 2025-11-04 08:56:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:56:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:45.479769+00:00 (in 27.999974 seconds) securityworker stdout | 2025-11-04 08:56:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:47 GMT)" (scheduled at 2025-11-04 08:56:17.479080+00:00) securityworker stdout | 2025-11-04 08:56:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:56:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:56:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:56:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityscanningnotificationworker stdout | 2025-11-04 08:56:17,486 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:56:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:56:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:47 GMT)" executed successfully reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:56:18,140 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:18,485 [288] [DEBUG] [app] Starting request: urn:request:3dd3e416-0507-4d0c-a3e3-eb8340bad12d (/api/v1/user/notifications) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 08:56:18,485 [288] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 08:56:18,485 [288] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 08:56:18,485 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 08:56:18,497 [288] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 08:56:18,497 [288] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 08:56:18,497 [288] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 08:56:18,497 [288] [DEBUG] [endpoints.api] Checking permission for user whuquay gunicorn-web stdout | 2025-11-04 08:56:18,497 [288] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 08:56:18,498 [288] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 08:56:18,498 [288] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 08:56:18,498 [288] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 08:56:18,499 [288] [DEBUG] [peewee] ('(SELECT "t1"."id", "t1"."uuid", "t1"."kind_id", "t1"."metadata_json", "t1"."dismissed", "t1"."lookup_path", "t1"."created", "t1"."created" AS "cd", "t1"."target_id" FROM "notification" AS "t1" INNER JOIN "notificationkind" AS "t2" ON ("t1"."kind_id" = "t2"."id") WHERE (("t1"."dismissed" = %s) AND ("t1"."target_id" = %s))) UNION (SELECT "t3"."id", "t3"."uuid", "t3"."kind_id", "t3"."metadata_json", "t3"."dismissed", "t3"."lookup_path", "t3"."created", "t3"."created" AS "cd", "t3"."target_id" FROM "notification" AS "t3" INNER JOIN "notificationkind" AS "t4" ON ("t3"."kind_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t5"."id" = "t3"."target_id") INNER JOIN "team" AS "t6" ON ("t5"."id" = "t6"."organization_id") INNER JOIN "teamrole" AS "t7" ON ("t6"."role_id" = "t7"."id") INNER JOIN "teammember" AS "t8" ON ("t6"."id" = "t8"."team_id") INNER JOIN "user" AS "t9" ON ("t8"."user_id" = "t9"."id") WHERE (("t3"."dismissed" = %s) AND (("t9"."id" = %s) AND ("t7"."name" = %s)))) ORDER BY cd desc LIMIT %s', [False, 1, False, 1, 'admin', 6]) gunicorn-web stdout | 2025-11-04 08:56:18,505 [288] [DEBUG] [app] Ending request: urn:request:3dd3e416-0507-4d0c-a3e3-eb8340bad12d (/api/v1/user/notifications) {'endpoint': 'api.usernotificationlist', 'request_id': 'urn:request:3dd3e416-0507-4d0c-a3e3-eb8340bad12d', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/user/notifications', 'path': '/api/v1/user/notifications', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 08:56:18,505 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:56:18,505 [288] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:08:56:18 +0000] "GET /api/v1/user/notifications HTTP/1.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:08:56:18 +0000] "GET /api/v1/user/notifications HTTP/2.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.021 956 0.022) repositorygcworker stdout | 2025-11-04 08:56:19,450 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:56:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:56:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:51.226294+00:00 (in 29.999560 seconds) autopruneworker stdout | 2025-11-04 08:56:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:51 GMT)" (scheduled at 2025-11-04 08:56:21.226294+00:00) autopruneworker stdout | 2025-11-04 08:56:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246521234, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:56:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:56:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:56:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 08:56:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:56:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:32.435813+00:00 (in 11.000959 seconds) namespacegcworker stdout | 2025-11-04 08:56:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:21 GMT)" (scheduled at 2025-11-04 08:56:21.434460+00:00) namespacegcworker stdout | 2025-11-04 08:56:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 08:56:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 21, 435136), True, datetime.datetime(2025, 11, 4, 8, 56, 21, 435136), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 08:56:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 08:56:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:56:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:56:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:56:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:31.530064+00:00 (in 9.999555 seconds) notificationworker stdout | 2025-11-04 08:56:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:31 GMT)" (scheduled at 2025-11-04 08:56:21.530064+00:00) notificationworker stdout | 2025-11-04 08:56:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:56:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 21, 530794), True, datetime.datetime(2025, 11, 4, 8, 56, 21, 530794), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:56:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:56:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:56:21,723 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:56:22,337 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:56:22,351 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:56:22,351 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:56:22,361 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:24,572 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:56:26,813 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:56:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:56:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:37.122849+00:00 (in 9.999609 seconds) proxycacheblobworker stdout | 2025-11-04 08:56:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:37 GMT)" (scheduled at 2025-11-04 08:56:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:56:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:56:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 27, 123551), True, datetime.datetime(2025, 11, 4, 8, 56, 27, 123551), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:56:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:56:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:56:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:56:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:56:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:57.222846+00:00 (in 29.999617 seconds) buildlogsarchiver stdout | 2025-11-04 08:56:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:57 GMT)" (scheduled at 2025-11-04 08:56:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:56:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 56, 27, 223418), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:56:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:56:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 08:56:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:56:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:56:27,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:56:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:56:27,528 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:56:28,176 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:56:28,567 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:28,832 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:28,987 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:29,103 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:29,127 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:29,183 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:56:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:56:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:44.351216+00:00 (in 14.999595 seconds) quotatotalworker stdout | 2025-11-04 08:56:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:44 GMT)" (scheduled at 2025-11-04 08:56:29.351216+00:00) quotatotalworker stdout | 2025-11-04 08:56:29,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:56:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:56:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:56:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:56:30,697 [287] [DEBUG] [app] Starting request: urn:request:75604dff-4ca6-4028-bab4-7d06ed5e0469 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:30,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:30,701 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:30,701 [292] [DEBUG] [app] Starting request: urn:request:c99e9b99-67ba-4089-a2d3-84eab5992e52 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:30,702 [292] [DEBUG] [app] Ending request: urn:request:c99e9b99-67ba-4089-a2d3-84eab5992e52 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c99e9b99-67ba-4089-a2d3-84eab5992e52', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:56:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 08:56:30,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:30,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:30,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:30,706 [285] [DEBUG] [app] Starting request: urn:request:aa6b5f89-ef48-49fa-b2fe-bf93af478d2b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:30,706 [285] [DEBUG] [app] Ending request: urn:request:aa6b5f89-ef48-49fa-b2fe-bf93af478d2b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:aa6b5f89-ef48-49fa-b2fe-bf93af478d2b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:56:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:56:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:30,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:30,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:30,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:30,715 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:30,715 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:30,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:56:30,728 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:56:30,731 [287] [DEBUG] [app] Ending request: urn:request:75604dff-4ca6-4028-bab4-7d06ed5e0469 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:75604dff-4ca6-4028-bab4-7d06ed5e0469', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:30,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:56:30,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 08:56:30,784 [288] [DEBUG] [app] Starting request: urn:request:edb65b51-938e-43ef-b61a-4238c7ca5e49 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:30,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:30,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:30,788 [298] [DEBUG] [app] Starting request: urn:request:9928d601-549e-4fb6-8c4e-2a9f4b861b6e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:30,789 [298] [DEBUG] [app] Ending request: urn:request:9928d601-549e-4fb6-8c4e-2a9f4b861b6e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9928d601-549e-4fb6-8c4e-2a9f4b861b6e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:56:30,789 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 08:56:30,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:30,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:30,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:30,793 [286] [DEBUG] [app] Starting request: urn:request:37572f6f-47a6-4ddc-a940-e0a79eafb435 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:30,793 [286] [DEBUG] [app] Ending request: urn:request:37572f6f-47a6-4ddc-a940-e0a79eafb435 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:37572f6f-47a6-4ddc-a940-e0a79eafb435', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:56:30,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 08:56:30,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:30,794 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:30,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:30,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:30,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:30,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:30,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:56:30,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:30,816 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:30,817 [288] [DEBUG] [app] Ending request: urn:request:edb65b51-938e-43ef-b61a-4238c7ca5e49 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:edb65b51-938e-43ef-b61a-4238c7ca5e49', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:30,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:56:30,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) notificationworker stdout | 2025-11-04 08:56:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:56:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:41.530064+00:00 (in 9.999607 seconds) notificationworker stdout | 2025-11-04 08:56:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:41 GMT)" (scheduled at 2025-11-04 08:56:31.530064+00:00) notificationworker stdout | 2025-11-04 08:56:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:56:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 31, 530700), True, datetime.datetime(2025, 11, 4, 8, 56, 31, 530700), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:56:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:56:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:41 GMT)" executed successfully gcworker stdout | 2025-11-04 08:56:31,617 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:56:31,805 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:56:31,821 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:56:32,361 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:56:32,376 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:56:32,376 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:56:32,386 [66] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:56:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:56:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:21.434460+00:00 (in 48.998227 seconds) namespacegcworker stdout | 2025-11-04 08:56:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:32 GMT)" (scheduled at 2025-11-04 08:56:32.435813+00:00) namespacegcworker stdout | 2025-11-04 08:56:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 08:56:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 08:56:33,092 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:56:33,103 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:56:33,208 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:56:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:56:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:48.625192+00:00 (in 15.001762 seconds) exportactionlogsworker stdout | 2025-11-04 08:56:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:33 GMT)" (scheduled at 2025-11-04 08:56:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 08:56:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 08:56:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 33, 623751), True, datetime.datetime(2025, 11, 4, 8, 56, 33, 623751), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 08:56:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 08:56:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 08:56:33,637 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:33 GMT)" executed successfully builder stdout | 2025-11-04 08:56:33,998 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:56:35,288 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:56:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:56:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:38.437104+00:00 (in 3.000906 seconds) gcworker stdout | 2025-11-04 08:56:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:05 GMT)" (scheduled at 2025-11-04 08:56:35.435809+00:00) gcworker stdout | 2025-11-04 08:56:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761036995436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:56:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:56:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:56:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:05 GMT)" executed successfully servicekey stdout | 2025-11-04 08:56:35,649 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:56:36,281 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:56:36,667 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:56:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:56:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:40.124445+00:00 (in 3.001180 seconds) proxycacheblobworker stdout | 2025-11-04 08:56:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:47 GMT)" (scheduled at 2025-11-04 08:56:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:56:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:56:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 37, 123535), True, datetime.datetime(2025, 11, 4, 8, 56, 37, 123535), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:56:37,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:56:37,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:56:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:56:37,400 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,449 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,467 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,474 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,469 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,489 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,543 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,647 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:56:37,677 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:56:37,947 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:56:37,950 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:56:37,967 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:56:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:56:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:05.435809+00:00 (in 26.998301 seconds) gcworker stdout | 2025-11-04 08:56:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:08 GMT)" (scheduled at 2025-11-04 08:56:38.437104+00:00) gcworker stdout | 2025-11-04 08:56:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:56:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246538449, None, 1, 0]) gcworker stdout | 2025-11-04 08:56:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:56:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:56:38,848 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:56:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:56:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:01.922827+00:00 (in 21.997770 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:56:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:39 GMT)" (scheduled at 2025-11-04 08:56:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:56:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 08:56:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 08:56:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:56:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:47.122849+00:00 (in 6.998016 seconds) proxycacheblobworker stdout | 2025-11-04 08:56:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:40 GMT)" (scheduled at 2025-11-04 08:56:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 08:56:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 08:56:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:56:40,562 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:56:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:56:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:51.530064+00:00 (in 9.999584 seconds) notificationworker stdout | 2025-11-04 08:56:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:51 GMT)" (scheduled at 2025-11-04 08:56:41.530064+00:00) notificationworker stdout | 2025-11-04 08:56:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:56:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 41, 530751), True, datetime.datetime(2025, 11, 4, 8, 56, 41, 530751), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:56:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:56:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:56:42,055 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:56:42,386 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:56:42,400 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:56:42,400 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:56:42,410 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:56:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:56:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:57.523915+00:00 (in 14.090619 seconds) repositorygcworker stdout | 2025-11-04 08:56:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:43 GMT)" (scheduled at 2025-11-04 08:56:43.432867+00:00) repositorygcworker stdout | 2025-11-04 08:56:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 08:56:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 43, 433582), True, datetime.datetime(2025, 11, 4, 8, 56, 43, 433582), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 08:56:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 08:56:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:56:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 08:56:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:56:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:59.351216+00:00 (in 14.999505 seconds) quotatotalworker stdout | 2025-11-04 08:56:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:59 GMT)" (scheduled at 2025-11-04 08:56:44.351216+00:00) quotatotalworker stdout | 2025-11-04 08:56:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:56:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:56:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:56:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:56:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:56:44,549 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:56:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:56:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:47.479080+00:00 (in 1.998894 seconds) securityworker stdout | 2025-11-04 08:56:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:15 GMT)" (scheduled at 2025-11-04 08:56:45.479769+00:00) securityworker stdout | 2025-11-04 08:56:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:56:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:56:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:56:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 51, 45, 483650), 1, 2]) securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 51, 45, 483650), 1, 2]) securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:56:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:56:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:15 GMT)" executed successfully securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker gunicorn-web stdout | 2025-11-04 08:56:45,697 [288] [DEBUG] [app] Starting request: urn:request:92c9f765-5e75-44de-baf5-4e9de42c22f0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:45,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:45,701 [292] [DEBUG] [app] Starting request: urn:request:96529a71-e4ad-48ce-adb7-8efe1c8f1b42 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:45,702 [292] [DEBUG] [app] Ending request: urn:request:96529a71-e4ad-48ce-adb7-8efe1c8f1b42 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:96529a71-e4ad-48ce-adb7-8efe1c8f1b42', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:56:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:56:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:45,706 [286] [DEBUG] [app] Starting request: urn:request:f31366bf-f327-42b3-ae76-47e347e33e5b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:45,706 [286] [DEBUG] [app] Ending request: urn:request:f31366bf-f327-42b3-ae76-47e347e33e5b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f31366bf-f327-42b3-ae76-47e347e33e5b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:56:45,706 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:56:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:45,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:56:45,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:56:45,731 [288] [DEBUG] [app] Ending request: urn:request:92c9f765-5e75-44de-baf5-4e9de42c22f0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:92c9f765-5e75-44de-baf5-4e9de42c22f0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:45,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:56:45,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:56:45,784 [285] [DEBUG] [app] Starting request: urn:request:45962bad-a808-4768-8d19-e33e8c6b8461 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:56:45,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:56:45,788 [292] [DEBUG] [app] Starting request: urn:request:2bb0914e-efb8-4ced-a84a-f465b179b4bf (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:56:45,788 [292] [DEBUG] [app] Ending request: urn:request:2bb0914e-efb8-4ced-a84a-f465b179b4bf (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2bb0914e-efb8-4ced-a84a-f465b179b4bf', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:56:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:56:45,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:45,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:56:45,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:56:45,793 [288] [DEBUG] [app] Starting request: urn:request:6d72f4fb-f810-4bc7-ac62-a90bbf885da3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:56:45,793 [288] [DEBUG] [app] Ending request: urn:request:6d72f4fb-f810-4bc7-ac62-a90bbf885da3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6d72f4fb-f810-4bc7-ac62-a90bbf885da3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:56:45,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:56:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:56:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:56:45,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:56:45,794 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:56:45,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:56:45,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:56:45,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:56:45,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:56:45,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:56:45,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:56:45,817 [285] [DEBUG] [app] Ending request: urn:request:45962bad-a808-4768-8d19-e33e8c6b8461 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:45962bad-a808-4768-8d19-e33e8c6b8461', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:56:45,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:56:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:56:45,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:56:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 08:56:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:56:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:57.122849+00:00 (in 9.999566 seconds) proxycacheblobworker stdout | 2025-11-04 08:56:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:57 GMT)" (scheduled at 2025-11-04 08:56:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:56:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:56:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 47, 123616), True, datetime.datetime(2025, 11, 4, 8, 56, 47, 123616), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:56:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:56:47,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:56:47,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:56:57 GMT)" executed successfully securityworker stdout | 2025-11-04 08:56:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:56:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:15.479769+00:00 (in 28.000207 seconds) securityworker stdout | 2025-11-04 08:56:47,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:17 GMT)" (scheduled at 2025-11-04 08:56:47.479080+00:00) securityworker stdout | 2025-11-04 08:56:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:56:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:56:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:56:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:56:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityscanningnotificationworker stdout | 2025-11-04 08:56:47,495 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 51, 47, 483538), 1, 2]) securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:56:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:56:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:56:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:56:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 51, 47, 483538), 1, 2]) securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:56:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:56:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:56:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:56:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:56:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:56:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:56:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:56:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:56:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:17 GMT)" executed successfully reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:56:48,149 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:56:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:56:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:56:58.624878+00:00 (in 9.999165 seconds) exportactionlogsworker stdout | 2025-11-04 08:56:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:48 GMT)" (scheduled at 2025-11-04 08:56:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 08:56:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 08:56:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:56:49,459 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:56:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:56:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:21.226294+00:00 (in 29.999571 seconds) autopruneworker stdout | 2025-11-04 08:56:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:21 GMT)" (scheduled at 2025-11-04 08:56:51.226294+00:00) autopruneworker stdout | 2025-11-04 08:56:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246551234, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:56:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:56:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:56:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:56:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:56:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:01.530064+00:00 (in 9.999582 seconds) notificationworker stdout | 2025-11-04 08:56:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:01 GMT)" (scheduled at 2025-11-04 08:56:51.530064+00:00) notificationworker stdout | 2025-11-04 08:56:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:56:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 51, 530748), True, datetime.datetime(2025, 11, 4, 8, 56, 51, 530748), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:56:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:56:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:56:51,732 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:56:52,410 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:56:52,425 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:56:52,425 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:56:52,435 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:56:54,581 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:56:56,822 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:56:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:56:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:07.122849+00:00 (in 9.999603 seconds) proxycacheblobworker stdout | 2025-11-04 08:56:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:07 GMT)" (scheduled at 2025-11-04 08:56:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:56:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:56:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 56, 57, 123547), True, datetime.datetime(2025, 11, 4, 8, 56, 57, 123547), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:56:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:09.133645+00:00 (in 12.001459 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:57 GMT)" (scheduled at 2025-11-04 08:56:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:56:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:56:57,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:56:57,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:56:57,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 08:56:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:56:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:56:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:56:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:56:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:27.222846+00:00 (in 29.999586 seconds) buildlogsarchiver stdout | 2025-11-04 08:56:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:27 GMT)" (scheduled at 2025-11-04 08:56:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:56:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 56, 57, 223468), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:56:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:56:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 08:56:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:56:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:56:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:56:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:56:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:43.432867+00:00 (in 45.908534 seconds) repositorygcworker stdout | 2025-11-04 08:56:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:57 GMT)" (scheduled at 2025-11-04 08:56:57.523915+00:00) repositorygcworker stdout | 2025-11-04 08:56:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 08:56:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:57:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:56:57,548 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:56:58,196 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:56:58,586 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:56:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:56:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:33.622985+00:00 (in 34.997697 seconds) exportactionlogsworker stdout | 2025-11-04 08:56:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:01:58 GMT)" (scheduled at 2025-11-04 08:56:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 08:56:58,625 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 56, 58, 625529), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 08:56:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 56, 58, 625529), True, datetime.datetime(2025, 11, 4, 8, 56, 58, 625529), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 08:56:58,641 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 56, 58, 625529), True, datetime.datetime(2025, 11, 4, 8, 56, 58, 625529), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 8, 56, 58, 625529), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 08:56:58,644 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 08:56:58,645 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:01:58 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:56:58,843 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:58,999 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:59,115 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:59,141 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:56:59,196 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:56:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:56:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:14.351216+00:00 (in 14.999525 seconds) quotatotalworker stdout | 2025-11-04 08:56:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:14 GMT)" (scheduled at 2025-11-04 08:56:59.351216+00:00) quotatotalworker stdout | 2025-11-04 08:56:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:56:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:56:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:56:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:57:00,697 [287] [DEBUG] [app] Starting request: urn:request:7674a67d-2c3f-4ca6-a15f-a40a67469c66 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:00,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:00,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:00,701 [292] [DEBUG] [app] Starting request: urn:request:91e510a9-e4fb-4b48-9536-3656f32224e4 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:00,702 [292] [DEBUG] [app] Ending request: urn:request:91e510a9-e4fb-4b48-9536-3656f32224e4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:91e510a9-e4fb-4b48-9536-3656f32224e4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 08:57:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:57:00,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:00,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:00,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:00,706 [285] [DEBUG] [app] Starting request: urn:request:ba6f0594-114a-4a22-b009-5ef890ec39d3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:00,706 [285] [DEBUG] [app] Ending request: urn:request:ba6f0594-114a-4a22-b009-5ef890ec39d3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ba6f0594-114a-4a22-b009-5ef890ec39d3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:57:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:57:00,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 08:57:00,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:00,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:00,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:00,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:00,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:00,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:00,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:00,730 [287] [DEBUG] [app] Ending request: urn:request:7674a67d-2c3f-4ca6-a15f-a40a67469c66 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7674a67d-2c3f-4ca6-a15f-a40a67469c66', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:00,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:57:00,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 08:57:00,784 [286] [DEBUG] [app] Starting request: urn:request:52883314-c8af-4c68-aa2f-7ac44f152300 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:00,785 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:00,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:00,788 [292] [DEBUG] [app] Starting request: urn:request:e06e7315-6d3d-42b6-811f-dfa5ee7de1f0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:00,788 [292] [DEBUG] [app] Ending request: urn:request:e06e7315-6d3d-42b6-811f-dfa5ee7de1f0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e06e7315-6d3d-42b6-811f-dfa5ee7de1f0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:57:00,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 08:57:00,789 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:00,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:00,792 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:00,792 [285] [DEBUG] [app] Starting request: urn:request:48504538-d12e-4bc8-9b07-a5492cb562ff (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:00,792 [285] [DEBUG] [app] Ending request: urn:request:48504538-d12e-4bc8-9b07-a5492cb562ff (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:48504538-d12e-4bc8-9b07-a5492cb562ff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:57:00,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:57:00,793 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:00,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:00,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:00,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:00,800 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:00,800 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:00,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:00,814 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:00,817 [286] [DEBUG] [app] Ending request: urn:request:52883314-c8af-4c68-aa2f-7ac44f152300 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:52883314-c8af-4c68-aa2f-7ac44f152300', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:00,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:57:00,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:00,824 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:57:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:57:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:11.530064+00:00 (in 9.999525 seconds) notificationworker stdout | 2025-11-04 08:57:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:11 GMT)" (scheduled at 2025-11-04 08:57:01.530064+00:00) notificationworker stdout | 2025-11-04 08:57:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:57:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 1, 530832), True, datetime.datetime(2025, 11, 4, 8, 57, 1, 530832), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:57:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:57:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:11 GMT)" executed successfully gcworker stdout | 2025-11-04 08:57:01,626 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:57:01,815 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:57:01,829 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:57:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:57:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:21.924304+00:00 (in 20.001120 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:57:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:01 GMT)" (scheduled at 2025-11-04 08:57:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:57:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 08:57:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 1, 923427), True, datetime.datetime(2025, 11, 4, 8, 57, 1, 923427), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 08:57:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 08:57:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 08:57:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:01 GMT)" executed successfully builder stdout | 2025-11-04 08:57:02,435 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:57:02,449 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:57:02,449 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:57:02,459 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:57:03,111 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:57:03,111 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:57:03,217 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:57:04,008 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:57:04,623 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run blobuploadcleanupworker stdout | 2025-11-04 08:57:04,624 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:57:04.623664+00:00 (in 3599.999535 seconds) blobuploadcleanupworker stdout | 2025-11-04 08:57:04,624 [65] [INFO] [apscheduler.executors.default] Running job "BlobUploadCleanupWorker._try_cleanup_uploads (trigger: interval[1:00:00], next run at: 2025-11-04 09:57:04 GMT)" (scheduled at 2025-11-04 08:57:04.623664+00:00) blobuploadcleanupworker stdout | 2025-11-04 08:57:04,624 [65] [DEBUG] [util.locking] Acquiring global lock BLOB_CLEANUP blobuploadcleanupworker stdout | 2025-11-04 08:57:04,624 [65] [DEBUG] [redis_lock.acquire] Getting 'lock:BLOB_CLEANUP' ... blobuploadcleanupworker stdout | 2025-11-04 08:57:04,625 [65] [INFO] [redis_lock.acquire] Got lock for 'lock:BLOB_CLEANUP'. blobuploadcleanupworker stdout | 2025-11-04 08:57:04,625 [65] [DEBUG] [util.locking] Acquired lock BLOB_CLEANUP blobuploadcleanupworker stdout | 2025-11-04 08:57:04,625 [65] [DEBUG] [__main__] Performing blob upload cleanup blobuploadcleanupworker stdout | 2025-11-04 08:57:04,625 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."created" <= %s) LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 2, 8, 57, 4, 625259), 1, 0]) blobuploadcleanupworker stdout | 2025-11-04 08:57:04,637 [65] [DEBUG] [__main__] No additional stale blob uploads found blobuploadcleanupworker stdout | 2025-11-04 08:57:04,637 [65] [DEBUG] [data.database] Disconnecting from database. blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.ListObjectsV2: calling handler > blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-call.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-call.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-call.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=ListObjectsV2) with params: {'url_path': '?list-type=2', 'query_string': {'prefix': 'quaydata/uploads', 'encoding-type': 'url'}, 'method': 'GET', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws?list-type=2&prefix=quaydata%2Fuploads&encoding-type=url', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 'encoding_type_auto_set': True, 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Prefix': 'quaydata/uploads', 'EncodingType': 'url'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event request-created.s3.ListObjectsV2: calling handler > blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event choose-signer.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,638 [65] [DEBUG] [botocore.hooks] Event before-sign.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.auth] Calculating signature using v4 auth. blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.auth] CanonicalRequest: blobuploadcleanupworker stdout | GET blobuploadcleanupworker stdout | /whuaws blobuploadcleanupworker stdout | encoding-type=url&list-type=2&prefix=quaydata%2Fuploads blobuploadcleanupworker stdout | host:s3.us-east-1.amazonaws.com blobuploadcleanupworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 blobuploadcleanupworker stdout | x-amz-date:20251104T085704Z blobuploadcleanupworker stdout | host;x-amz-content-sha256;x-amz-date blobuploadcleanupworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.auth] StringToSign: blobuploadcleanupworker stdout | AWS4-HMAC-SHA256 blobuploadcleanupworker stdout | 20251104T085704Z blobuploadcleanupworker stdout | 20251104/us-east-1/s3/aws4_request blobuploadcleanupworker stdout | 288a5255305660d212d2e4a44651a2f655ae6992320aee9dfb3599f00a6a8cbe blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.auth] Signature: blobuploadcleanupworker stdout | e88f2146f5b9bf533b771a045288d88b863c57a02fe2fd26cf9c7542e26e96a7 blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.hooks] Event request-created.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.endpoint] Sending http request: blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.hooks] Event before-send.s3.ListObjectsV2: calling handler > blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T085704Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e88f2146f5b9bf533b771a045288d88b863c57a02fe2fd26cf9c7542e26e96a7', 'amz-sdk-invocation-id': b'4926659a-686b-442f-a518-850114d51c2d', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem blobuploadcleanupworker stdout | 2025-11-04 08:57:04,639 [65] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: s3.us-east-1.amazonaws.com blobuploadcleanupworker stdout | 2025-11-04 08:57:04,739 [65] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "GET /whuaws?list-type=2&prefix=quaydata%2Fuploads&encoding-type=url HTTP/1.1" 200 None blobuploadcleanupworker stdout | 2025-11-04 08:57:04,739 [65] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'kuq0Ypf8X7VnSmuN2elXBmS3yBhvzlg9LinZPcQsOoWvT8lLSfBcwV8B32LC9jJjjHC/QG5U8Hg=', 'x-amz-request-id': 'RRKQ17PHY5AG30RY', 'Date': 'Tue, 04 Nov 2025 08:57:05 GMT', 'x-amz-bucket-region': 'us-east-1', 'Content-Type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} blobuploadcleanupworker stdout | 2025-11-04 08:57:04,740 [65] [DEBUG] [botocore.parsers] Response body: blobuploadcleanupworker stdout | b'\nwhuawsquaydata/uploads31000urlfalsequaydata/uploads/02538aea-3fe9-498e-92d3-437a8e182a372025-11-04T07:17:40.000Z"22a5764c6a5618898d1eb48516b5f722-1"CRC64NVMEFULL_OBJECT7830STANDARDquaydata/uploads/3c288da2-ae84-4cc1-8373-f7610c935bf12025-11-04T07:17:33.000Z"24f177ab924bfa4bf5868712ebad9856-4"CRC64NVMEFULL_OBJECT20144818STANDARDquaydata/uploads/dc202c6c-3507-45d8-9217-9ce5b6fe2a4d2025-11-04T07:17:34.000Z"4acd48c660e031f0432360c0077839cd-16"CRC64NVMEFULL_OBJECT80005019STANDARD' blobuploadcleanupworker stdout | 2025-11-04 08:57:04,740 [65] [DEBUG] [botocore.hooks] Event needs-retry.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,740 [65] [DEBUG] [botocore.retryhandler] No retry needed. blobuploadcleanupworker stdout | 2025-11-04 08:57:04,740 [65] [DEBUG] [botocore.hooks] Event needs-retry.s3.ListObjectsV2: calling handler > blobuploadcleanupworker stdout | 2025-11-04 08:57:04,740 [65] [DEBUG] [botocore.hooks] Event after-call.s3.ListObjectsV2: calling handler > blobuploadcleanupworker stdout | 2025-11-04 08:57:04,741 [65] [DEBUG] [botocore.hooks] Event after-call.s3.ListObjectsV2: calling handler blobuploadcleanupworker stdout | 2025-11-04 08:57:04,741 [65] [DEBUG] [util.locking] Releasing lock BLOB_CLEANUP blobuploadcleanupworker stdout | 2025-11-04 08:57:04,741 [65] [DEBUG] [redis_lock.release] Releasing 'lock:BLOB_CLEANUP'. blobuploadcleanupworker stdout | 2025-11-04 08:57:04,741 [65] [DEBUG] [util.locking] Released lock BLOB_CLEANUP blobuploadcleanupworker stdout | 2025-11-04 08:57:04,741 [65] [INFO] [apscheduler.executors.default] Job "BlobUploadCleanupWorker._try_cleanup_uploads (trigger: interval[1:00:00], next run at: 2025-11-04 09:57:04 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:57:05,296 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:57:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:57:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:08.437104+00:00 (in 3.000902 seconds) gcworker stdout | 2025-11-04 08:57:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:35 GMT)" (scheduled at 2025-11-04 08:57:05.435809+00:00) gcworker stdout | 2025-11-04 08:57:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037025436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:57:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:57:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:57:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:35 GMT)" executed successfully servicekey stdout | 2025-11-04 08:57:05,658 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:57:06,291 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:57:06,676 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:57:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:57:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:17.122849+00:00 (in 9.999567 seconds) proxycacheblobworker stdout | 2025-11-04 08:57:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:17 GMT)" (scheduled at 2025-11-04 08:57:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:57:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:57:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 7, 123591), True, datetime.datetime(2025, 11, 4, 8, 57, 7, 123591), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:57:07,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:57:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:57:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:57:07,411 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,460 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,479 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,497 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,499 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,488 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,555 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,658 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:07,690 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:57:07,973 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:57:07,976 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:57:07,978 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:57:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:57:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:35.435809+00:00 (in 26.998248 seconds) gcworker stdout | 2025-11-04 08:57:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:38 GMT)" (scheduled at 2025-11-04 08:57:08.437104+00:00) gcworker stdout | 2025-11-04 08:57:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:57:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246568449, None, 1, 0]) gcworker stdout | 2025-11-04 08:57:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:57:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:57:08,857 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 08:57:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:57:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:57.131795+00:00 (in 47.997728 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:09 GMT)" (scheduled at 2025-11-04 08:57:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:57:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:57:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:57:10,570 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:57:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:57:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:15.535095+00:00 (in 4.004596 seconds) notificationworker stdout | 2025-11-04 08:57:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:21 GMT)" (scheduled at 2025-11-04 08:57:11.530064+00:00) notificationworker stdout | 2025-11-04 08:57:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:57:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 11, 530760), True, datetime.datetime(2025, 11, 4, 8, 57, 11, 530760), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:57:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:57:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:57:12,064 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:57:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 08:57:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:12.222828+00:00 (in 59.999553 seconds) quotaregistrysizeworker stdout | 2025-11-04 08:57:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:12 GMT)" (scheduled at 2025-11-04 08:57:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 08:57:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 08:57:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 08:57:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:12 GMT)" executed successfully builder stdout | 2025-11-04 08:57:12,459 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:57:12,474 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:57:12,474 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:57:12,484 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:57:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:57:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:29.351216+00:00 (in 14.999555 seconds) quotatotalworker stdout | 2025-11-04 08:57:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:29 GMT)" (scheduled at 2025-11-04 08:57:14.351216+00:00) quotatotalworker stdout | 2025-11-04 08:57:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:57:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:57:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:57:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:57:14,558 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:57:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:57:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:17.479080+00:00 (in 1.998882 seconds) securityworker stdout | 2025-11-04 08:57:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:45 GMT)" (scheduled at 2025-11-04 08:57:15.479769+00:00) securityworker stdout | 2025-11-04 08:57:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:57:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:57:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:57:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:57:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 52, 15, 483807), 1, 2]) securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 52, 15, 483807), 1, 2]) securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:57:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:57:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:21.530064+00:00 (in 5.994525 seconds) notificationworker stdout | 2025-11-04 08:57:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:15 GMT)" (scheduled at 2025-11-04 08:57:15.535095+00:00) notificationworker stdout | 2025-11-04 08:57:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 08:57:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:57:15,697 [288] [DEBUG] [app] Starting request: urn:request:bb167b6b-8d1b-46c4-b1f7-fbe07f97ddcf (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:15,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:15,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:15,702 [297] [DEBUG] [app] Starting request: urn:request:929474ed-de13-4824-aaaa-62fc3f31ffa8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:15,702 [297] [DEBUG] [app] Ending request: urn:request:929474ed-de13-4824-aaaa-62fc3f31ffa8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:929474ed-de13-4824-aaaa-62fc3f31ffa8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:57:15,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 08:57:15,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:15,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:15,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:15,706 [286] [DEBUG] [app] Starting request: urn:request:fbabf674-f269-46c4-8d1e-3745f40bd8d8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:15,706 [286] [DEBUG] [app] Ending request: urn:request:fbabf674-f269-46c4-8d1e-3745f40bd8d8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:fbabf674-f269-46c4-8d1e-3745f40bd8d8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:57:15,707 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 08:57:15,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:15,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:15,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:15,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:15,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:15,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:15,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:15,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:15,731 [288] [DEBUG] [app] Ending request: urn:request:bb167b6b-8d1b-46c4-b1f7-fbe07f97ddcf (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bb167b6b-8d1b-46c4-b1f7-fbe07f97ddcf', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:15,731 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:57:15,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 08:57:15,783 [285] [DEBUG] [app] Starting request: urn:request:c025f4b3-5f16-4be6-97b7-03b11cb48b9d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:15,788 [299] [DEBUG] [app] Starting request: urn:request:76a6ea7b-a8b7-4b40-a35b-7b4719408355 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:15,788 [299] [DEBUG] [app] Ending request: urn:request:76a6ea7b-a8b7-4b40-a35b-7b4719408355 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:76a6ea7b-a8b7-4b40-a35b-7b4719408355', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:57:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:57:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:15,792 [288] [DEBUG] [app] Starting request: urn:request:33083419-1de3-4c32-b30c-b3501de10691 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:15,793 [288] [DEBUG] [app] Ending request: urn:request:33083419-1de3-4c32-b30c-b3501de10691 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:33083419-1de3-4c32-b30c-b3501de10691', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:57:15,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:57:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:15,817 [285] [DEBUG] [app] Ending request: urn:request:c025f4b3-5f16-4be6-97b7-03b11cb48b9d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c025f4b3-5f16-4be6-97b7-03b11cb48b9d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:57:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) proxycacheblobworker stdout | 2025-11-04 08:57:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:57:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:27.122849+00:00 (in 9.999587 seconds) proxycacheblobworker stdout | 2025-11-04 08:57:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:27 GMT)" (scheduled at 2025-11-04 08:57:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:57:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:57:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 17, 123537), True, datetime.datetime(2025, 11, 4, 8, 57, 17, 123537), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:57:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:57:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:57:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:27 GMT)" executed successfully securityworker stdout | 2025-11-04 08:57:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:57:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:45.479769+00:00 (in 28.000211 seconds) securityworker stdout | 2025-11-04 08:57:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:47 GMT)" (scheduled at 2025-11-04 08:57:17.479080+00:00) securityworker stdout | 2025-11-04 08:57:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:57:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:57:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:57:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:57:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:57:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:57:17,504 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:57:18,158 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 08:57:19,469 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:57:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:57:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:51.226294+00:00 (in 29.999544 seconds) autopruneworker stdout | 2025-11-04 08:57:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:51 GMT)" (scheduled at 2025-11-04 08:57:21.226294+00:00) autopruneworker stdout | 2025-11-04 08:57:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246581235, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:57:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:57:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:57:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 08:57:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:57:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:32.435813+00:00 (in 11.000898 seconds) namespacegcworker stdout | 2025-11-04 08:57:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:21 GMT)" (scheduled at 2025-11-04 08:57:21.434460+00:00) namespacegcworker stdout | 2025-11-04 08:57:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 08:57:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 21, 435239), True, datetime.datetime(2025, 11, 4, 8, 57, 21, 435239), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 08:57:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 08:57:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:57:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:57:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:57:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:31.530064+00:00 (in 9.999607 seconds) notificationworker stdout | 2025-11-04 08:57:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:31 GMT)" (scheduled at 2025-11-04 08:57:21.530064+00:00) notificationworker stdout | 2025-11-04 08:57:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:57:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 21, 530722), True, datetime.datetime(2025, 11, 4, 8, 57, 21, 530722), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:57:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:57:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:57:21,742 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:57:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:57:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:39.924652+00:00 (in 17.999931 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:57:21,924 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:02:21 GMT)" (scheduled at 2025-11-04 08:57:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:57:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 57, 21, 924976), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 08:57:21,937 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 57, 21, 924976), True, datetime.datetime(2025, 11, 4, 8, 57, 21, 924976), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 08:57:21,941 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 57, 21, 924976), True, datetime.datetime(2025, 11, 4, 8, 57, 21, 924976), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 8, 57, 21, 924976), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 08:57:21,944 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 08:57:21,944 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:02:21 GMT)" executed successfully builder stdout | 2025-11-04 08:57:22,484 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:57:22,498 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:57:22,498 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:57:22,508 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:24,591 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:57:26,830 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:57:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:57:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:37.122849+00:00 (in 9.999573 seconds) proxycacheblobworker stdout | 2025-11-04 08:57:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:37 GMT)" (scheduled at 2025-11-04 08:57:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:57:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:57:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 27, 123573), True, datetime.datetime(2025, 11, 4, 8, 57, 27, 123573), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:57:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:57:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:57:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:57:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:57:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:57.222846+00:00 (in 29.999620 seconds) buildlogsarchiver stdout | 2025-11-04 08:57:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:57 GMT)" (scheduled at 2025-11-04 08:57:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:57:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 57, 27, 223453), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:57:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:57:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 08:57:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:57:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:57:27,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:57:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:57:27,557 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:57:28,205 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:57:28,595 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:28,853 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:29,012 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:29,129 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:29,155 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:29,209 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:57:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:57:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:44.351216+00:00 (in 14.999546 seconds) quotatotalworker stdout | 2025-11-04 08:57:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:44 GMT)" (scheduled at 2025-11-04 08:57:29.351216+00:00) quotatotalworker stdout | 2025-11-04 08:57:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:57:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:57:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:57:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:57:30,697 [287] [DEBUG] [app] Starting request: urn:request:44474732-eae0-40b4-8f15-5fef6450bba8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:30,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:30,701 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:30,701 [292] [DEBUG] [app] Starting request: urn:request:e42acd6c-28d8-454d-9202-e595b7cb6f75 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:30,702 [292] [DEBUG] [app] Ending request: urn:request:e42acd6c-28d8-454d-9202-e595b7cb6f75 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e42acd6c-28d8-454d-9202-e595b7cb6f75', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 08:57:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:57:30,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:30,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:30,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:30,706 [285] [DEBUG] [app] Starting request: urn:request:f5fe993f-975b-4957-b86a-dc18aa3dad8c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:30,706 [285] [DEBUG] [app] Ending request: urn:request:f5fe993f-975b-4957-b86a-dc18aa3dad8c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f5fe993f-975b-4957-b86a-dc18aa3dad8c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:57:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:57:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:30,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:30,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:30,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:30,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:30,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:30,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:30,728 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:30,731 [287] [DEBUG] [app] Ending request: urn:request:44474732-eae0-40b4-8f15-5fef6450bba8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:44474732-eae0-40b4-8f15-5fef6450bba8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:30,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:57:30,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:57:30,783 [286] [DEBUG] [app] Starting request: urn:request:56bc828b-9fb6-404e-8af8-f57599c039bb (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:30,784 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:30,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:30,788 [292] [DEBUG] [app] Starting request: urn:request:fd1e5453-ef07-4c82-a525-9071f52a51d4 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:30,788 [292] [DEBUG] [app] Ending request: urn:request:fd1e5453-ef07-4c82-a525-9071f52a51d4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fd1e5453-ef07-4c82-a525-9071f52a51d4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:57:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:57:30,788 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:30,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:30,791 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:30,792 [285] [DEBUG] [app] Starting request: urn:request:97bc2e3f-ec79-4373-984d-241bc6857f65 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:30,792 [285] [DEBUG] [app] Ending request: urn:request:97bc2e3f-ec79-4373-984d-241bc6857f65 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:97bc2e3f-ec79-4373-984d-241bc6857f65', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:57:30,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:57:30,792 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:30,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:30,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:30,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:30,800 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:30,800 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:30,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:30,814 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:30,817 [286] [DEBUG] [app] Ending request: urn:request:56bc828b-9fb6-404e-8af8-f57599c039bb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:56bc828b-9fb6-404e-8af8-f57599c039bb', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:30,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:57:30,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:30,833 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:57:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:57:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:41.530064+00:00 (in 9.999491 seconds) notificationworker stdout | 2025-11-04 08:57:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:41 GMT)" (scheduled at 2025-11-04 08:57:31.530064+00:00) notificationworker stdout | 2025-11-04 08:57:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:57:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 31, 530899), True, datetime.datetime(2025, 11, 4, 8, 57, 31, 530899), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:57:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:57:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:41 GMT)" executed successfully gcworker stdout | 2025-11-04 08:57:31,635 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:57:31,824 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:57:31,838 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:57:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:57:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:08.435432+00:00 (in 35.999192 seconds) namespacegcworker stdout | 2025-11-04 08:57:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:32 GMT)" (scheduled at 2025-11-04 08:57:32.435813+00:00) namespacegcworker stdout | 2025-11-04 08:57:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 08:57:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:32 GMT)" executed successfully builder stdout | 2025-11-04 08:57:32,508 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:57:32,521 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:57:32,521 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:57:32,532 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:57:33,120 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:57:33,132 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:57:33,226 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:57:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:57:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:48.625192+00:00 (in 15.001785 seconds) exportactionlogsworker stdout | 2025-11-04 08:57:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:33 GMT)" (scheduled at 2025-11-04 08:57:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 08:57:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 08:57:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 33, 623740), True, datetime.datetime(2025, 11, 4, 8, 57, 33, 623740), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 08:57:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 08:57:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 08:57:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:33 GMT)" executed successfully builder stdout | 2025-11-04 08:57:34,018 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:57:35,305 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:57:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:57:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:38.437104+00:00 (in 3.000902 seconds) gcworker stdout | 2025-11-04 08:57:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:05 GMT)" (scheduled at 2025-11-04 08:57:35.435809+00:00) gcworker stdout | 2025-11-04 08:57:35,436 [72] [DEBUG] [peewee] ('SELECT DISTINCT "t1"."removed_tag_expiration_s" FROM "user" AS "t1" LIMIT %s', [100]) gcworker stdout | 2025-11-04 08:57:35,448 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037055448, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:57:35,453 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:57:35,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:57:35,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:05 GMT)" executed successfully servicekey stdout | 2025-11-04 08:57:35,667 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:57:36,301 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:57:36,685 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:57:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:57:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:40.124445+00:00 (in 3.001168 seconds) proxycacheblobworker stdout | 2025-11-04 08:57:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:47 GMT)" (scheduled at 2025-11-04 08:57:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:57:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:57:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 37, 123602), True, datetime.datetime(2025, 11, 4, 8, 57, 37, 123602), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:57:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:57:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:57:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:57:37,430 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,471 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,505 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,513 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,514 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,518 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,565 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,669 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:57:37,702 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:57:37,984 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:57:37,988 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:57:37,988 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:57:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:57:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:05.435809+00:00 (in 26.998307 seconds) gcworker stdout | 2025-11-04 08:57:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:08 GMT)" (scheduled at 2025-11-04 08:57:38.437104+00:00) gcworker stdout | 2025-11-04 08:57:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:57:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246598449, None, 1, 0]) gcworker stdout | 2025-11-04 08:57:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:57:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:57:38,866 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:57:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:57:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:01.922827+00:00 (in 21.997779 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:57:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:39 GMT)" (scheduled at 2025-11-04 08:57:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:57:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 08:57:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 08:57:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:57:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:47.122849+00:00 (in 6.997996 seconds) proxycacheblobworker stdout | 2025-11-04 08:57:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:40 GMT)" (scheduled at 2025-11-04 08:57:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 08:57:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 08:57:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:57:40,579 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:57:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:57:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:51.530064+00:00 (in 9.999504 seconds) notificationworker stdout | 2025-11-04 08:57:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:51 GMT)" (scheduled at 2025-11-04 08:57:41.530064+00:00) notificationworker stdout | 2025-11-04 08:57:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:57:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 41, 530830), True, datetime.datetime(2025, 11, 4, 8, 57, 41, 530830), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:57:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:57:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:57:42,073 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:57:42,532 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:57:42,547 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:57:42,547 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:57:42,556 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:57:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:57:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:57.523915+00:00 (in 14.090652 seconds) repositorygcworker stdout | 2025-11-04 08:57:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:43 GMT)" (scheduled at 2025-11-04 08:57:43.432867+00:00) repositorygcworker stdout | 2025-11-04 08:57:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 08:57:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 43, 433575), True, datetime.datetime(2025, 11, 4, 8, 57, 43, 433575), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 08:57:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 08:57:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:57:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 08:57:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:57:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:59.351216+00:00 (in 14.999530 seconds) quotatotalworker stdout | 2025-11-04 08:57:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:59 GMT)" (scheduled at 2025-11-04 08:57:44.351216+00:00) quotatotalworker stdout | 2025-11-04 08:57:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:57:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:57:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:57:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:57:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:57:44,575 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:57:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:57:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:47.479080+00:00 (in 1.998842 seconds) securityworker stdout | 2025-11-04 08:57:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:15 GMT)" (scheduled at 2025-11-04 08:57:45.479769+00:00) securityworker stdout | 2025-11-04 08:57:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:57:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:57:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:57:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:57:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 52, 45, 483840), 1, 2]) securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 52, 45, 483840), 1, 2]) securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:57:45,697 [286] [DEBUG] [app] Starting request: urn:request:a580f684-8395-4ff3-b254-b418be304d80 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:45,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:45,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:45,702 [292] [DEBUG] [app] Starting request: urn:request:d9d703db-a485-4e72-880d-fff42894d30c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:45,702 [292] [DEBUG] [app] Ending request: urn:request:d9d703db-a485-4e72-880d-fff42894d30c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d9d703db-a485-4e72-880d-fff42894d30c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:57:45,703 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 08:57:45,703 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:45,704 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:45,706 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:45,707 [285] [DEBUG] [app] Starting request: urn:request:4cacfad6-3f0e-4eb1-97e4-3956c335f021 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:45,707 [285] [DEBUG] [app] Ending request: urn:request:4cacfad6-3f0e-4eb1-97e4-3956c335f021 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4cacfad6-3f0e-4eb1-97e4-3956c335f021', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:57:45,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:57:45,707 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:45,708 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:45,708 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:45,708 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:45,715 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:45,715 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:45,725 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:45,728 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:45,731 [286] [DEBUG] [app] Ending request: urn:request:a580f684-8395-4ff3-b254-b418be304d80 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a580f684-8395-4ff3-b254-b418be304d80', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:45,732 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 08:57:45,732 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 08:57:45,784 [286] [DEBUG] [app] Starting request: urn:request:4e7b4a7a-ecd6-41ae-89dd-9e9a7f2c6a6c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:57:45,785 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:57:45,785 [296] [DEBUG] [app] Starting request: urn:request:e0d6e241-3add-4134-ad4c-9a0ff0eaf1c1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:57:45,786 [296] [DEBUG] [app] Ending request: urn:request:e0d6e241-3add-4134-ad4c-9a0ff0eaf1c1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e0d6e241-3add-4134-ad4c-9a0ff0eaf1c1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 08:57:45,786 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:57:45,786 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:45,787 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:57:45,789 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:57:45,790 [285] [DEBUG] [app] Starting request: urn:request:a3342881-e9c7-4db4-8ef7-306b4146c6b3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:57:45,790 [285] [DEBUG] [app] Ending request: urn:request:a3342881-e9c7-4db4-8ef7-306b4146c6b3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a3342881-e9c7-4db4-8ef7-306b4146c6b3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:57:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:57:45,790 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:57:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:57:45,790 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:57:45,791 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:57:45,791 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:57:45,791 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:57:45,798 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:57:45,798 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:57:45,808 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:57:45,811 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:57:45,814 [286] [DEBUG] [app] Ending request: urn:request:4e7b4a7a-ecd6-41ae-89dd-9e9a7f2c6a6c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4e7b4a7a-ecd6-41ae-89dd-9e9a7f2c6a6c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:57:45,814 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:57:45,814 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:57:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:57:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) proxycacheblobworker stdout | 2025-11-04 08:57:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:57:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:57:57.122849+00:00 (in 9.999578 seconds) proxycacheblobworker stdout | 2025-11-04 08:57:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:57 GMT)" (scheduled at 2025-11-04 08:57:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:57:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:57:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 47, 123577), True, datetime.datetime(2025, 11, 4, 8, 57, 47, 123577), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:57:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:57:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:57:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:57:57 GMT)" executed successfully securityworker stdout | 2025-11-04 08:57:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:57:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:15.479769+00:00 (in 28.000209 seconds) securityworker stdout | 2025-11-04 08:57:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:17 GMT)" (scheduled at 2025-11-04 08:57:47.479080+00:00) securityworker stdout | 2025-11-04 08:57:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:57:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:57:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:57:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:57:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:57:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 52, 47, 483304), 1, 2]) securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:57:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 52, 47, 483304), 1, 2]) securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:57:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:57:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:57:47,513 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:57:48,167 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:57:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:57:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:33.622985+00:00 (in 44.997371 seconds) exportactionlogsworker stdout | 2025-11-04 08:57:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:48 GMT)" (scheduled at 2025-11-04 08:57:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 08:57:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 08:57:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:57:49,478 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:57:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:57:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:21.226294+00:00 (in 29.999513 seconds) autopruneworker stdout | 2025-11-04 08:57:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:21 GMT)" (scheduled at 2025-11-04 08:57:51.226294+00:00) autopruneworker stdout | 2025-11-04 08:57:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246611235, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:57:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:57:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:57:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:57:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:57:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:01.530064+00:00 (in 9.999605 seconds) notificationworker stdout | 2025-11-04 08:57:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:01 GMT)" (scheduled at 2025-11-04 08:57:51.530064+00:00) notificationworker stdout | 2025-11-04 08:57:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:57:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 51, 530712), True, datetime.datetime(2025, 11, 4, 8, 57, 51, 530712), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:57:51,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:57:51,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:57:51,751 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:57:52,557 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:57:52,570 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:57:52,570 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:57:52,580 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:57:54,600 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:57:56,839 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:57:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:57:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:07.122849+00:00 (in 9.999600 seconds) proxycacheblobworker stdout | 2025-11-04 08:57:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:07 GMT)" (scheduled at 2025-11-04 08:57:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:57:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:57:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 57, 57, 123553), True, datetime.datetime(2025, 11, 4, 8, 57, 57, 123553), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:57:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:09.133645+00:00 (in 12.001449 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:57 GMT)" (scheduled at 2025-11-04 08:57:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:57:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:57:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:57:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:57:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 08:57:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:57:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:57:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:57:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:57:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:27.222846+00:00 (in 29.999616 seconds) buildlogsarchiver stdout | 2025-11-04 08:57:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:27 GMT)" (scheduled at 2025-11-04 08:57:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:57:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 57, 57, 223443), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:57:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:57:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 08:57:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:57:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:57:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:57:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:57:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:43.432867+00:00 (in 45.908509 seconds) repositorygcworker stdout | 2025-11-04 08:57:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:57 GMT)" (scheduled at 2025-11-04 08:57:57.523915+00:00) repositorygcworker stdout | 2025-11-04 08:57:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 08:57:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:58:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:57:57,567 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:57:58,213 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:57:58,604 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:58,864 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:59,023 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:59,143 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:59,168 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:57:59,223 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:57:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:57:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:14.351216+00:00 (in 14.999543 seconds) quotatotalworker stdout | 2025-11-04 08:57:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:14 GMT)" (scheduled at 2025-11-04 08:57:59.351216+00:00) quotatotalworker stdout | 2025-11-04 08:57:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:57:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:57:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:57:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:58:00,697 [288] [DEBUG] [app] Starting request: urn:request:7c6008f2-dde7-43e5-aea1-f04034c2267d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:00,701 [292] [DEBUG] [app] Starting request: urn:request:6f8473b0-829e-4211-ad94-6b5fed715bb7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:00,702 [292] [DEBUG] [app] Ending request: urn:request:6f8473b0-829e-4211-ad94-6b5fed715bb7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6f8473b0-829e-4211-ad94-6b5fed715bb7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:58:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:58:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:00,706 [285] [DEBUG] [app] Starting request: urn:request:ced5f797-f81f-4d96-b6b9-21371e52f248 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:00,706 [285] [DEBUG] [app] Ending request: urn:request:ced5f797-f81f-4d96-b6b9-21371e52f248 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ced5f797-f81f-4d96-b6b9-21371e52f248', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:58:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:58:00,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:00,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:00,730 [288] [DEBUG] [app] Ending request: urn:request:7c6008f2-dde7-43e5-aea1-f04034c2267d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7c6008f2-dde7-43e5-aea1-f04034c2267d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:00,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:58:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:58:00,784 [285] [DEBUG] [app] Starting request: urn:request:5189f9b5-82a5-4818-a99f-16fced24feba (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:00,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:00,788 [297] [DEBUG] [app] Starting request: urn:request:84df749c-a9bb-462c-b6fb-3f7ba4352486 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:00,789 [297] [DEBUG] [app] Ending request: urn:request:84df749c-a9bb-462c-b6fb-3f7ba4352486 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:84df749c-a9bb-462c-b6fb-3f7ba4352486', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:58:00,789 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 08:58:00,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:00,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:00,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:00,792 [288] [DEBUG] [app] Starting request: urn:request:6fcd7558-21a4-4ef4-996b-89a8043753bd (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:00,793 [288] [DEBUG] [app] Ending request: urn:request:6fcd7558-21a4-4ef4-996b-89a8043753bd (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6fcd7558-21a4-4ef4-996b-89a8043753bd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:58:00,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:58:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:00,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:00,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:00,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:00,816 [285] [DEBUG] [app] Ending request: urn:request:5189f9b5-82a5-4818-a99f-16fced24feba (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5189f9b5-82a5-4818-a99f-16fced24feba', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:00,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:58:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:00,842 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:58:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:58:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:11.530064+00:00 (in 9.999532 seconds) notificationworker stdout | 2025-11-04 08:58:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:11 GMT)" (scheduled at 2025-11-04 08:58:01.530064+00:00) notificationworker stdout | 2025-11-04 08:58:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:58:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 1, 530796), True, datetime.datetime(2025, 11, 4, 8, 58, 1, 530796), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:58:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:58:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:11 GMT)" executed successfully gcworker stdout | 2025-11-04 08:58:01,644 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:58:01,832 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:58:01,847 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:58:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:58:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:39.924652+00:00 (in 38.001412 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:58:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:01 GMT)" (scheduled at 2025-11-04 08:58:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:58:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 08:58:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 1, 923556), True, datetime.datetime(2025, 11, 4, 8, 58, 1, 923556), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 08:58:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 08:58:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 08:58:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:01 GMT)" executed successfully builder stdout | 2025-11-04 08:58:02,580 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:58:02,594 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:58:02,594 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:58:02,604 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:58:03,129 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:58:03,141 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:58:03,234 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:58:04,027 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:58:05,314 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:58:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:58:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:08.437104+00:00 (in 3.000924 seconds) gcworker stdout | 2025-11-04 08:58:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:35 GMT)" (scheduled at 2025-11-04 08:58:05.435809+00:00) gcworker stdout | 2025-11-04 08:58:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037085436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:58:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:58:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:58:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:35 GMT)" executed successfully servicekey stdout | 2025-11-04 08:58:05,676 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:58:06,310 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:58:06,694 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:58:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:58:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:17.122849+00:00 (in 9.999592 seconds) proxycacheblobworker stdout | 2025-11-04 08:58:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:17 GMT)" (scheduled at 2025-11-04 08:58:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:58:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:58:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 7, 123557), True, datetime.datetime(2025, 11, 4, 8, 58, 7, 123557), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:58:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:58:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:58:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:58:07,470 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,481 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,516 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,527 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,527 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,530 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,577 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,680 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:07,713 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:58:07,996 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:58:08,000 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:58:08,000 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:58:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:58:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:21.434460+00:00 (in 12.998605 seconds) namespacegcworker stdout | 2025-11-04 08:58:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:03:08 GMT)" (scheduled at 2025-11-04 08:58:08.435432+00:00) namespacegcworker stdout | 2025-11-04 08:58:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 58, 8, 436100), 'namespacegc/%']) gcworker stdout | 2025-11-04 08:58:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:58:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:35.435809+00:00 (in 26.998347 seconds) gcworker stdout | 2025-11-04 08:58:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:38 GMT)" (scheduled at 2025-11-04 08:58:08.437104+00:00) gcworker stdout | 2025-11-04 08:58:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) namespacegcworker stdout | 2025-11-04 08:58:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 58, 8, 436100), True, datetime.datetime(2025, 11, 4, 8, 58, 8, 436100), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 08:58:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246628449, None, 1, 0]) namespacegcworker stdout | 2025-11-04 08:58:08,452 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 58, 8, 436100), True, datetime.datetime(2025, 11, 4, 8, 58, 8, 436100), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 8, 58, 8, 436100), 'namespacegc/%']) gcworker stdout | 2025-11-04 08:58:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:58:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 08:58:08,455 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:58:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:03:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:58:08,875 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 08:58:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:58:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:57.131795+00:00 (in 47.997717 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:09 GMT)" (scheduled at 2025-11-04 08:58:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:58:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:58:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:58:10,588 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:58:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:58:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:15.535095+00:00 (in 4.004564 seconds) notificationworker stdout | 2025-11-04 08:58:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:21 GMT)" (scheduled at 2025-11-04 08:58:11.530064+00:00) notificationworker stdout | 2025-11-04 08:58:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:58:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 11, 530871), True, datetime.datetime(2025, 11, 4, 8, 58, 11, 530871), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:58:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:58:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:58:12,087 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:58:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 08:58:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:12.222828+00:00 (in 59.999549 seconds) quotaregistrysizeworker stdout | 2025-11-04 08:58:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:12 GMT)" (scheduled at 2025-11-04 08:58:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 08:58:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 08:58:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 08:58:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:12 GMT)" executed successfully builder stdout | 2025-11-04 08:58:12,604 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:58:12,618 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:58:12,618 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:58:12,628 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:58:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:58:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:29.351216+00:00 (in 14.999577 seconds) quotatotalworker stdout | 2025-11-04 08:58:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:29 GMT)" (scheduled at 2025-11-04 08:58:14.351216+00:00) quotatotalworker stdout | 2025-11-04 08:58:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:58:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:58:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:58:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:58:14,584 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:58:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:58:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:17.479080+00:00 (in 1.998860 seconds) securityworker stdout | 2025-11-04 08:58:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:45 GMT)" (scheduled at 2025-11-04 08:58:15.479769+00:00) securityworker stdout | 2025-11-04 08:58:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:58:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:58:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:58:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:58:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 53, 15, 483749), 1, 2]) securityworker stdout | 2025-11-04 08:58:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 53, 15, 483749), 1, 2]) securityworker stdout | 2025-11-04 08:58:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:58:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:58:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:21.530064+00:00 (in 5.994520 seconds) notificationworker stdout | 2025-11-04 08:58:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:15 GMT)" (scheduled at 2025-11-04 08:58:15.535095+00:00) notificationworker stdout | 2025-11-04 08:58:15,536 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 08:58:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:58:15,696 [285] [DEBUG] [app] Starting request: urn:request:354b7379-1b39-40ac-abf2-783f1ddd2956 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:15,701 [299] [DEBUG] [app] Starting request: urn:request:b3d35e94-44c6-4e7b-b908-43b32ec0657f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:15,701 [299] [DEBUG] [app] Ending request: urn:request:b3d35e94-44c6-4e7b-b908-43b32ec0657f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b3d35e94-44c6-4e7b-b908-43b32ec0657f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:58:15,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:58:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:15,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:15,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:15,705 [288] [DEBUG] [app] Starting request: urn:request:1bfe89c5-0775-41df-9601-439192ddc3ca (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:15,705 [288] [DEBUG] [app] Ending request: urn:request:1bfe89c5-0775-41df-9601-439192ddc3ca (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1bfe89c5-0775-41df-9601-439192ddc3ca', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:58:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:58:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:15,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:15,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:15,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:15,730 [285] [DEBUG] [app] Ending request: urn:request:354b7379-1b39-40ac-abf2-783f1ddd2956 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:354b7379-1b39-40ac-abf2-783f1ddd2956', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:15,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:58:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:58:15,783 [285] [DEBUG] [app] Starting request: urn:request:94e480d7-ea3f-4026-af0a-1bca523a1c65 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:15,785 [292] [DEBUG] [app] Starting request: urn:request:b43db4b3-0dad-4000-9f43-9245429e43b7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:15,785 [292] [DEBUG] [app] Ending request: urn:request:b43db4b3-0dad-4000-9f43-9245429e43b7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b43db4b3-0dad-4000-9f43-9245429e43b7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:58:15,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 08:58:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:15,789 [288] [DEBUG] [app] Starting request: urn:request:df319ccc-1010-44eb-b8ed-6c4dca45cf65 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:15,790 [288] [DEBUG] [app] Ending request: urn:request:df319ccc-1010-44eb-b8ed-6c4dca45cf65 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:df319ccc-1010-44eb-b8ed-6c4dca45cf65', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:58:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:58:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:15,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:15,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:15,814 [285] [DEBUG] [app] Ending request: urn:request:94e480d7-ea3f-4026-af0a-1bca523a1c65 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:94e480d7-ea3f-4026-af0a-1bca523a1c65', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:15,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:58:15,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.032) proxycacheblobworker stdout | 2025-11-04 08:58:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:58:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:27.122849+00:00 (in 9.999634 seconds) proxycacheblobworker stdout | 2025-11-04 08:58:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:27 GMT)" (scheduled at 2025-11-04 08:58:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:58:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:58:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 17, 123453), True, datetime.datetime(2025, 11, 4, 8, 58, 17, 123453), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:58:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:58:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:58:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:27 GMT)" executed successfully securityworker stdout | 2025-11-04 08:58:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:58:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:45.479769+00:00 (in 28.000232 seconds) securityworker stdout | 2025-11-04 08:58:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:47 GMT)" (scheduled at 2025-11-04 08:58:17.479080+00:00) securityworker stdout | 2025-11-04 08:58:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:58:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:58:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:58:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:58:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:58:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:58:17,522 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:58:18,176 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 08:58:19,487 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:58:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:58:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:51.226294+00:00 (in 29.999543 seconds) autopruneworker stdout | 2025-11-04 08:58:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:51 GMT)" (scheduled at 2025-11-04 08:58:21.226294+00:00) autopruneworker stdout | 2025-11-04 08:58:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246641235, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:58:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:58:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:58:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 08:58:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:58:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:32.435813+00:00 (in 11.000904 seconds) namespacegcworker stdout | 2025-11-04 08:58:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:21 GMT)" (scheduled at 2025-11-04 08:58:21.434460+00:00) namespacegcworker stdout | 2025-11-04 08:58:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 08:58:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 21, 435235), True, datetime.datetime(2025, 11, 4, 8, 58, 21, 435235), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 08:58:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 08:58:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:58:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:58:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:58:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:31.530064+00:00 (in 9.999643 seconds) notificationworker stdout | 2025-11-04 08:58:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:31 GMT)" (scheduled at 2025-11-04 08:58:21.530064+00:00) notificationworker stdout | 2025-11-04 08:58:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:58:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 21, 530704), True, datetime.datetime(2025, 11, 4, 8, 58, 21, 530704), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:58:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:58:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:21,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:58:21,760 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:58:22,628 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:58:22,642 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:58:22,642 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:58:22,652 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:24,609 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:58:26,848 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:58:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:58:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:37.122849+00:00 (in 9.999593 seconds) proxycacheblobworker stdout | 2025-11-04 08:58:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:37 GMT)" (scheduled at 2025-11-04 08:58:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:58:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:58:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 27, 123553), True, datetime.datetime(2025, 11, 4, 8, 58, 27, 123553), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:58:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:58:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:58:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:58:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:58:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:57.222846+00:00 (in 29.999600 seconds) buildlogsarchiver stdout | 2025-11-04 08:58:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:57 GMT)" (scheduled at 2025-11-04 08:58:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:58:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 58, 27, 223473), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:58:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:58:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 08:58:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:58:27,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:58:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:58:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:58:27,576 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:58:28,222 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:58:28,613 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:28,875 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:29,039 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:29,156 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:29,181 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:29,236 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:58:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:58:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:44.351216+00:00 (in 14.999588 seconds) quotatotalworker stdout | 2025-11-04 08:58:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:44 GMT)" (scheduled at 2025-11-04 08:58:29.351216+00:00) quotatotalworker stdout | 2025-11-04 08:58:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:58:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:58:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:58:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:58:30,697 [286] [DEBUG] [app] Starting request: urn:request:2b9228de-fd98-4271-8ad5-490606914d38 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:30,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:30,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:30,702 [292] [DEBUG] [app] Starting request: urn:request:b3872049-ba65-4bf7-b5bc-a96535b67cd0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:30,702 [292] [DEBUG] [app] Ending request: urn:request:b3872049-ba65-4bf7-b5bc-a96535b67cd0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b3872049-ba65-4bf7-b5bc-a96535b67cd0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 08:58:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:58:30,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:30,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:30,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:30,706 [285] [DEBUG] [app] Starting request: urn:request:512b938e-4aa9-40a4-9cfa-4567d6da28f4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:30,706 [285] [DEBUG] [app] Ending request: urn:request:512b938e-4aa9-40a4-9cfa-4567d6da28f4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:512b938e-4aa9-40a4-9cfa-4567d6da28f4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:58:30,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:58:30,707 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 08:58:30,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:30,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:30,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:30,715 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:30,715 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:30,725 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:30,728 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:30,731 [286] [DEBUG] [app] Ending request: urn:request:2b9228de-fd98-4271-8ad5-490606914d38 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2b9228de-fd98-4271-8ad5-490606914d38', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:30,732 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) gunicorn-web stdout | 2025-11-04 08:58:30,732 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 08:58:30,783 [288] [DEBUG] [app] Starting request: urn:request:121fdfd9-91c3-4e9d-9c98-583668f27783 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:30,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:30,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:30,787 [299] [DEBUG] [app] Starting request: urn:request:30c07d12-0e7c-4377-8699-6983cb9b4512 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:30,788 [299] [DEBUG] [app] Ending request: urn:request:30c07d12-0e7c-4377-8699-6983cb9b4512 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:30c07d12-0e7c-4377-8699-6983cb9b4512', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:58:30,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:58:30,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:30,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:30,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:30,792 [285] [DEBUG] [app] Starting request: urn:request:74c67402-7e93-4795-abd7-1f124153e58d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:30,792 [285] [DEBUG] [app] Ending request: urn:request:74c67402-7e93-4795-abd7-1f124153e58d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:74c67402-7e93-4795-abd7-1f124153e58d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:58:30,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 08:58:30,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:30,792 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:30,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:30,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:30,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:30,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:30,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:30,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:30,816 [288] [DEBUG] [app] Ending request: urn:request:121fdfd9-91c3-4e9d-9c98-583668f27783 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:121fdfd9-91c3-4e9d-9c98-583668f27783', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:30,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:58:30,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:30,851 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:58:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:58:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:41.530064+00:00 (in 9.999515 seconds) notificationworker stdout | 2025-11-04 08:58:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:41 GMT)" (scheduled at 2025-11-04 08:58:31.530064+00:00) notificationworker stdout | 2025-11-04 08:58:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:58:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 31, 530843), True, datetime.datetime(2025, 11, 4, 8, 58, 31, 530843), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:58:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:58:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:41 GMT)" executed successfully gcworker stdout | 2025-11-04 08:58:31,653 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:58:31,853 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:58:31,855 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:58:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:58:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:21.434460+00:00 (in 48.998215 seconds) namespacegcworker stdout | 2025-11-04 08:58:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:32 GMT)" (scheduled at 2025-11-04 08:58:32.435813+00:00) namespacegcworker stdout | 2025-11-04 08:58:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 08:58:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:32 GMT)" executed successfully builder stdout | 2025-11-04 08:58:32,652 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:58:32,668 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:58:32,668 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:58:32,678 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:58:33,138 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:58:33,150 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:58:33,243 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:58:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:58:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:48.625192+00:00 (in 15.001761 seconds) exportactionlogsworker stdout | 2025-11-04 08:58:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:33 GMT)" (scheduled at 2025-11-04 08:58:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 08:58:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 08:58:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 33, 623787), True, datetime.datetime(2025, 11, 4, 8, 58, 33, 623787), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 08:58:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 08:58:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 08:58:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:33 GMT)" executed successfully builder stdout | 2025-11-04 08:58:34,036 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:58:35,323 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:58:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:58:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:38.437104+00:00 (in 3.000876 seconds) gcworker stdout | 2025-11-04 08:58:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:05 GMT)" (scheduled at 2025-11-04 08:58:35.435809+00:00) gcworker stdout | 2025-11-04 08:58:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037115436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:58:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:58:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:58:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:05 GMT)" executed successfully servicekey stdout | 2025-11-04 08:58:35,685 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:58:36,320 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:58:36,703 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:58:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:58:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:40.124445+00:00 (in 3.001172 seconds) proxycacheblobworker stdout | 2025-11-04 08:58:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:47 GMT)" (scheduled at 2025-11-04 08:58:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:58:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:58:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 37, 123581), True, datetime.datetime(2025, 11, 4, 8, 58, 37, 123581), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:58:37,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:58:37,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:58:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:58:37,484 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,495 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,526 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,539 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,540 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,542 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,591 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,696 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:58:37,724 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:58:38,008 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:58:38,009 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:58:38,011 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:58:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:58:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:05.435809+00:00 (in 26.998231 seconds) gcworker stdout | 2025-11-04 08:58:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:08 GMT)" (scheduled at 2025-11-04 08:58:38.437104+00:00) gcworker stdout | 2025-11-04 08:58:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:58:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246658448, None, 1, 0]) gcworker stdout | 2025-11-04 08:58:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:58:38,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:58:38,885 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:58:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:58:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:01.922827+00:00 (in 21.997788 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:58:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:39 GMT)" (scheduled at 2025-11-04 08:58:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:58:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 08:58:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 08:58:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:58:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:47.122849+00:00 (in 6.997981 seconds) proxycacheblobworker stdout | 2025-11-04 08:58:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:40 GMT)" (scheduled at 2025-11-04 08:58:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 08:58:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 08:58:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:58:40,597 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:58:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:58:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:51.530064+00:00 (in 9.999542 seconds) notificationworker stdout | 2025-11-04 08:58:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:51 GMT)" (scheduled at 2025-11-04 08:58:41.530064+00:00) notificationworker stdout | 2025-11-04 08:58:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:58:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 41, 530754), True, datetime.datetime(2025, 11, 4, 8, 58, 41, 530754), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:58:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:58:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:58:42,096 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:58:42,678 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:58:42,691 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:58:42,691 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:58:42,708 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:58:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:58:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:57.523915+00:00 (in 14.090640 seconds) repositorygcworker stdout | 2025-11-04 08:58:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:43 GMT)" (scheduled at 2025-11-04 08:58:43.432867+00:00) repositorygcworker stdout | 2025-11-04 08:58:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 08:58:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 43, 433593), True, datetime.datetime(2025, 11, 4, 8, 58, 43, 433593), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 08:58:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 08:58:43,445 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:58:43,445 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 08:58:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:58:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:59.351216+00:00 (in 14.999570 seconds) quotatotalworker stdout | 2025-11-04 08:58:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:59 GMT)" (scheduled at 2025-11-04 08:58:44.351216+00:00) quotatotalworker stdout | 2025-11-04 08:58:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:58:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:58:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:58:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:58:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:58:44,593 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:58:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:58:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:47.479080+00:00 (in 1.998848 seconds) securityworker stdout | 2025-11-04 08:58:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:15 GMT)" (scheduled at 2025-11-04 08:58:45.479769+00:00) securityworker stdout | 2025-11-04 08:58:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:58:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:58:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:58:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:58:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:58:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:58:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 53, 45, 484122), 1, 2]) securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 53, 45, 484122), 1, 2]) securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:58:45,696 [285] [DEBUG] [app] Starting request: urn:request:1a7b1ee8-345b-4a21-ad4d-67f323980065 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:45,701 [297] [DEBUG] [app] Starting request: urn:request:635892ed-37b2-40e7-b83e-c0ec8c11c778 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:45,701 [297] [DEBUG] [app] Ending request: urn:request:635892ed-37b2-40e7-b83e-c0ec8c11c778 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:635892ed-37b2-40e7-b83e-c0ec8c11c778', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:58:45,701 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:58:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:45,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:45,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:45,705 [288] [DEBUG] [app] Starting request: urn:request:6d52cd19-9619-4a2d-8303-51431b284750 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:45,705 [288] [DEBUG] [app] Ending request: urn:request:6d52cd19-9619-4a2d-8303-51431b284750 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6d52cd19-9619-4a2d-8303-51431b284750', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:58:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:58:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:45,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:45,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:45,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:45,730 [285] [DEBUG] [app] Ending request: urn:request:1a7b1ee8-345b-4a21-ad4d-67f323980065 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1a7b1ee8-345b-4a21-ad4d-67f323980065', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:45,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:58:45,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:58:45,783 [285] [DEBUG] [app] Starting request: urn:request:bf944fd9-4734-4662-9a08-482e069348e0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:58:45,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:58:45,785 [292] [DEBUG] [app] Starting request: urn:request:ac0caa63-e589-427f-bcf8-c13fe95797fd (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:58:45,785 [292] [DEBUG] [app] Ending request: urn:request:ac0caa63-e589-427f-bcf8-c13fe95797fd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ac0caa63-e589-427f-bcf8-c13fe95797fd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:58:45,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:58:45,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:45,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:58:45,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:58:45,789 [288] [DEBUG] [app] Starting request: urn:request:b314cce2-0720-430b-87ee-18a76f22a42a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:58:45,789 [288] [DEBUG] [app] Ending request: urn:request:b314cce2-0720-430b-87ee-18a76f22a42a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b314cce2-0720-430b-87ee-18a76f22a42a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:58:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:58:45,789 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:58:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:58:45,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:58:45,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:58:45,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:58:45,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:58:45,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:58:45,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:58:45,806 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:58:45,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:58:45,813 [285] [DEBUG] [app] Ending request: urn:request:bf944fd9-4734-4662-9a08-482e069348e0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bf944fd9-4734-4662-9a08-482e069348e0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:58:45,813 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:58:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.030 47 0.030) gunicorn-web stdout | 2025-11-04 08:58:45,813 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:58:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 08:58:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:58:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:58:57.122849+00:00 (in 9.999638 seconds) proxycacheblobworker stdout | 2025-11-04 08:58:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:57 GMT)" (scheduled at 2025-11-04 08:58:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:58:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:58:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 47, 123467), True, datetime.datetime(2025, 11, 4, 8, 58, 47, 123467), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:58:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:58:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:58:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:58:57 GMT)" executed successfully securityworker stdout | 2025-11-04 08:58:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:58:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:15.479769+00:00 (in 28.000205 seconds) securityworker stdout | 2025-11-04 08:58:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:17 GMT)" (scheduled at 2025-11-04 08:58:47.479080+00:00) securityworker stdout | 2025-11-04 08:58:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:58:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:58:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:58:47,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:58:47,496 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:47,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:47,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:47,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:58:47,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:58:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:47,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 53, 47, 484122), 1, 2]) securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:58:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:58:47,508 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 53, 47, 484122), 1, 2]) securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:58:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:58:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:58:47,531 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:58:48,185 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:58:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:58:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:33.622985+00:00 (in 44.997301 seconds) exportactionlogsworker stdout | 2025-11-04 08:58:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:48 GMT)" (scheduled at 2025-11-04 08:58:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 08:58:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 08:58:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:58:49,497 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:58:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:58:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:21.226294+00:00 (in 29.999578 seconds) autopruneworker stdout | 2025-11-04 08:58:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:21 GMT)" (scheduled at 2025-11-04 08:58:51.226294+00:00) autopruneworker stdout | 2025-11-04 08:58:51,234 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246671234, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:58:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:58:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:58:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:58:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:58:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:01.530064+00:00 (in 9.999514 seconds) notificationworker stdout | 2025-11-04 08:58:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:01 GMT)" (scheduled at 2025-11-04 08:58:51.530064+00:00) notificationworker stdout | 2025-11-04 08:58:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:58:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 51, 530834), True, datetime.datetime(2025, 11, 4, 8, 58, 51, 530834), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:58:51,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:58:51,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:58:51,780 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:58:52,708 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:58:52,725 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:58:52,725 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:58:52,735 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:58:54,618 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:58:56,857 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:58:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:58:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:07.122849+00:00 (in 9.999620 seconds) proxycacheblobworker stdout | 2025-11-04 08:58:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:07 GMT)" (scheduled at 2025-11-04 08:58:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:58:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:58:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 58, 57, 123527), True, datetime.datetime(2025, 11, 4, 8, 58, 57, 123527), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:58:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:09.133645+00:00 (in 12.001457 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:57 GMT)" (scheduled at 2025-11-04 08:58:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:58:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:58:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:58:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:58:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 08:58:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:58:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:58:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:58:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:58:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:27.222846+00:00 (in 29.999625 seconds) buildlogsarchiver stdout | 2025-11-04 08:58:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:27 GMT)" (scheduled at 2025-11-04 08:58:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:58:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 58, 57, 223420), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:58:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:58:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 08:58:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:58:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:58:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:58:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:58:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:02.523412+00:00 (in 4.998964 seconds) repositorygcworker stdout | 2025-11-04 08:58:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:57 GMT)" (scheduled at 2025-11-04 08:58:57.523915+00:00) repositorygcworker stdout | 2025-11-04 08:58:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 08:58:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 08:59:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:58:57,584 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:58:58,232 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:58:58,622 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:58,886 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:59,051 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:59,170 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:59,195 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:58:59,250 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:58:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:58:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:14.351216+00:00 (in 14.999543 seconds) quotatotalworker stdout | 2025-11-04 08:58:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:14 GMT)" (scheduled at 2025-11-04 08:58:59.351216+00:00) quotatotalworker stdout | 2025-11-04 08:58:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:58:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:58:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:58:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:59:00,697 [285] [DEBUG] [app] Starting request: urn:request:cb04b2d3-a5c4-4ec0-8578-b6ecb4f40c9f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:00,702 [298] [DEBUG] [app] Starting request: urn:request:edb3a7bc-4a8c-41b2-83e5-509ff3a94311 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:00,702 [298] [DEBUG] [app] Ending request: urn:request:edb3a7bc-4a8c-41b2-83e5-509ff3a94311 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:edb3a7bc-4a8c-41b2-83e5-509ff3a94311', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:59:00,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:59:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:00,706 [287] [DEBUG] [app] Starting request: urn:request:5a355b6b-712b-45f3-b890-398ea025a714 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:00,706 [287] [DEBUG] [app] Ending request: urn:request:5a355b6b-712b-45f3-b890-398ea025a714 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5a355b6b-712b-45f3-b890-398ea025a714', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:59:00,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:59:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 08:59:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:00,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:00,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:00,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:00,731 [285] [DEBUG] [app] Ending request: urn:request:cb04b2d3-a5c4-4ec0-8578-b6ecb4f40c9f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:cb04b2d3-a5c4-4ec0-8578-b6ecb4f40c9f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:00,731 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:59:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 08:59:00,783 [288] [DEBUG] [app] Starting request: urn:request:81f8a4f8-a69e-4bbd-a57c-cbd4ddc40df8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:00,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:00,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:00,788 [298] [DEBUG] [app] Starting request: urn:request:24e6527e-c9aa-416c-a2d7-13d1c224369e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:00,788 [298] [DEBUG] [app] Ending request: urn:request:24e6527e-c9aa-416c-a2d7-13d1c224369e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:24e6527e-c9aa-416c-a2d7-13d1c224369e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 08:59:00,788 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:59:00,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:00,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:00,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:00,792 [286] [DEBUG] [app] Starting request: urn:request:f34e0a2b-ce45-43c3-927a-ee1369b7d7c8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:00,792 [286] [DEBUG] [app] Ending request: urn:request:f34e0a2b-ce45-43c3-927a-ee1369b7d7c8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f34e0a2b-ce45-43c3-927a-ee1369b7d7c8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:59:00,792 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:59:00,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:00,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:00,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:00,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:00,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:00,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:00,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:00,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:00,816 [288] [DEBUG] [app] Ending request: urn:request:81f8a4f8-a69e-4bbd-a57c-cbd4ddc40df8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:81f8a4f8-a69e-4bbd-a57c-cbd4ddc40df8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:00,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:59:00,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:00,860 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:59:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:11.530064+00:00 (in 9.999474 seconds) notificationworker stdout | 2025-11-04 08:59:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:11 GMT)" (scheduled at 2025-11-04 08:59:01.530064+00:00) notificationworker stdout | 2025-11-04 08:59:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:59:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 1, 530872), True, datetime.datetime(2025, 11, 4, 8, 59, 1, 530872), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:59:01,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:59:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:11 GMT)" executed successfully gcworker stdout | 2025-11-04 08:59:01,662 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:59:01,862 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:59:01,864 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:59:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:59:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:39.924652+00:00 (in 38.001416 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:59:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:01 GMT)" (scheduled at 2025-11-04 08:59:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:59:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 08:59:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 1, 923544), True, datetime.datetime(2025, 11, 4, 8, 59, 1, 923544), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 08:59:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 08:59:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 08:59:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:01 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:59:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:59:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:43.432867+00:00 (in 40.908992 seconds) repositorygcworker stdout | 2025-11-04 08:59:02,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:04:02 GMT)" (scheduled at 2025-11-04 08:59:02.523412+00:00) repositorygcworker stdout | 2025-11-04 08:59:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 59, 2, 524132), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 08:59:02,536 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 59, 2, 524132), True, datetime.datetime(2025, 11, 4, 8, 59, 2, 524132), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 08:59:02,539 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 59, 2, 524132), True, datetime.datetime(2025, 11, 4, 8, 59, 2, 524132), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 8, 59, 2, 524132), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 08:59:02,543 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:59:02,543 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:04:02 GMT)" executed successfully builder stdout | 2025-11-04 08:59:02,735 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:59:02,749 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:59:02,749 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:59:02,759 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:59:03,147 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:59:03,159 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:59:03,252 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:59:04,046 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:59:05,332 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:59:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:59:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:08.437104+00:00 (in 3.000894 seconds) gcworker stdout | 2025-11-04 08:59:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:35 GMT)" (scheduled at 2025-11-04 08:59:05.435809+00:00) gcworker stdout | 2025-11-04 08:59:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037145436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:59:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:59:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:59:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:35 GMT)" executed successfully servicekey stdout | 2025-11-04 08:59:05,694 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:59:06,329 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:59:06,723 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:59:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:17.122849+00:00 (in 9.999600 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:17 GMT)" (scheduled at 2025-11-04 08:59:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:59:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 7, 123561), True, datetime.datetime(2025, 11, 4, 8, 59, 7, 123561), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:59:07,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:59:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:59:07,498 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,508 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,541 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,549 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,551 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,553 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,604 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,711 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:07,739 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:59:08,019 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:59:08,020 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:59:08,021 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:59:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:59:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:35.435809+00:00 (in 26.998247 seconds) gcworker stdout | 2025-11-04 08:59:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:38 GMT)" (scheduled at 2025-11-04 08:59:08.437104+00:00) gcworker stdout | 2025-11-04 08:59:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:59:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246688449, None, 1, 0]) gcworker stdout | 2025-11-04 08:59:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:59:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:59:08,894 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 08:59:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:59:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:57.131795+00:00 (in 47.997718 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:09 GMT)" (scheduled at 2025-11-04 08:59:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:09,147 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:59:09,147 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:59:09,147 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:59:10,605 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:59:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:15.535095+00:00 (in 4.004505 seconds) notificationworker stdout | 2025-11-04 08:59:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:21 GMT)" (scheduled at 2025-11-04 08:59:11.530064+00:00) notificationworker stdout | 2025-11-04 08:59:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:59:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 11, 530894), True, datetime.datetime(2025, 11, 4, 8, 59, 11, 530894), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:59:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:59:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:59:12,105 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:59:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 08:59:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:12.222828+00:00 (in 59.999595 seconds) quotaregistrysizeworker stdout | 2025-11-04 08:59:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:12 GMT)" (scheduled at 2025-11-04 08:59:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 08:59:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 08:59:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 08:59:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:12 GMT)" executed successfully builder stdout | 2025-11-04 08:59:12,759 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:59:12,772 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:59:12,772 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:59:12,782 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:59:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:59:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:29.351216+00:00 (in 14.999555 seconds) quotatotalworker stdout | 2025-11-04 08:59:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:29 GMT)" (scheduled at 2025-11-04 08:59:14.351216+00:00) quotatotalworker stdout | 2025-11-04 08:59:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:59:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:59:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:59:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:59:14,602 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:59:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:59:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:17.479080+00:00 (in 1.998849 seconds) securityworker stdout | 2025-11-04 08:59:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:45 GMT)" (scheduled at 2025-11-04 08:59:15.479769+00:00) securityworker stdout | 2025-11-04 08:59:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:59:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:59:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:59:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:59:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:59:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:59:15,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 54, 15, 483763), 1, 2]) securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:59:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 54, 15, 483763), 1, 2]) securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:59:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:59:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:21.530064+00:00 (in 5.994495 seconds) notificationworker stdout | 2025-11-04 08:59:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:15 GMT)" (scheduled at 2025-11-04 08:59:15.535095+00:00) notificationworker stdout | 2025-11-04 08:59:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 08:59:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:59:15,697 [286] [DEBUG] [app] Starting request: urn:request:9c0f3660-ad15-41b8-8a47-6e57a770d7e2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:15,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:15,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:15,702 [292] [DEBUG] [app] Starting request: urn:request:e7edbb87-95a4-47fb-bf0b-4e0d31366c8f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:15,702 [292] [DEBUG] [app] Ending request: urn:request:e7edbb87-95a4-47fb-bf0b-4e0d31366c8f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e7edbb87-95a4-47fb-bf0b-4e0d31366c8f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:59:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 08:59:15,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:15,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:15,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:15,706 [285] [DEBUG] [app] Starting request: urn:request:76056db1-6505-488b-aa25-598e54f8cd3e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:15,706 [285] [DEBUG] [app] Ending request: urn:request:76056db1-6505-488b-aa25-598e54f8cd3e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:76056db1-6505-488b-aa25-598e54f8cd3e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:59:15,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:59:15,707 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:15,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:15,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:15,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:15,715 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:15,715 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:15,725 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:15,728 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:15,731 [286] [DEBUG] [app] Ending request: urn:request:9c0f3660-ad15-41b8-8a47-6e57a770d7e2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9c0f3660-ad15-41b8-8a47-6e57a770d7e2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:15,731 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:59:15,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 08:59:15,784 [285] [DEBUG] [app] Starting request: urn:request:bb7efa39-dbdd-4c01-9ddb-babadb5fb276 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:15,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:15,788 [292] [DEBUG] [app] Starting request: urn:request:c9ea763d-a5d8-41e3-8dad-c8a825006230 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:15,788 [292] [DEBUG] [app] Ending request: urn:request:c9ea763d-a5d8-41e3-8dad-c8a825006230 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c9ea763d-a5d8-41e3-8dad-c8a825006230', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:59:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:59:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:15,792 [288] [DEBUG] [app] Starting request: urn:request:3937e7b5-a5b9-47c9-aabe-de1fbf4918b3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:15,792 [288] [DEBUG] [app] Ending request: urn:request:3937e7b5-a5b9-47c9-aabe-de1fbf4918b3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3937e7b5-a5b9-47c9-aabe-de1fbf4918b3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:59:15,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:59:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:15,817 [285] [DEBUG] [app] Ending request: urn:request:bb7efa39-dbdd-4c01-9ddb-babadb5fb276 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bb7efa39-dbdd-4c01-9ddb-babadb5fb276', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:59:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 08:59:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:27.122849+00:00 (in 9.999584 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:27 GMT)" (scheduled at 2025-11-04 08:59:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:59:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 17, 123565), True, datetime.datetime(2025, 11, 4, 8, 59, 17, 123565), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:59:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:59:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:27 GMT)" executed successfully securityworker stdout | 2025-11-04 08:59:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:59:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:45.479769+00:00 (in 28.000204 seconds) securityworker stdout | 2025-11-04 08:59:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:47 GMT)" (scheduled at 2025-11-04 08:59:17.479080+00:00) securityworker stdout | 2025-11-04 08:59:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:59:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:59:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:59:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:59:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 08:59:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:59:17,539 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:59:18,194 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 08:59:19,506 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:59:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:59:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:51.226294+00:00 (in 29.999552 seconds) autopruneworker stdout | 2025-11-04 08:59:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:51 GMT)" (scheduled at 2025-11-04 08:59:21.226294+00:00) autopruneworker stdout | 2025-11-04 08:59:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246701235, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:59:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:59:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:59:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 08:59:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:59:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:32.435813+00:00 (in 11.000921 seconds) namespacegcworker stdout | 2025-11-04 08:59:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:21 GMT)" (scheduled at 2025-11-04 08:59:21.434460+00:00) namespacegcworker stdout | 2025-11-04 08:59:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 08:59:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 21, 435186), True, datetime.datetime(2025, 11, 4, 8, 59, 21, 435186), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 08:59:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 08:59:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 08:59:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:59:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:31.530064+00:00 (in 9.999535 seconds) notificationworker stdout | 2025-11-04 08:59:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:31 GMT)" (scheduled at 2025-11-04 08:59:21.530064+00:00) notificationworker stdout | 2025-11-04 08:59:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:59:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 21, 530812), True, datetime.datetime(2025, 11, 4, 8, 59, 21, 530812), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:59:21,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:59:21,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:21,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:59:21,789 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:59:22,782 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:59:22,796 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:59:22,796 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:59:22,806 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:24,628 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:59:26,876 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:59:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:37.122849+00:00 (in 9.999619 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:37 GMT)" (scheduled at 2025-11-04 08:59:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:59:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 27, 123473), True, datetime.datetime(2025, 11, 4, 8, 59, 27, 123473), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:59:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:59:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:59:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:59:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:57.222846+00:00 (in 29.999659 seconds) buildlogsarchiver stdout | 2025-11-04 08:59:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:57 GMT)" (scheduled at 2025-11-04 08:59:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:59:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 59, 27, 223374), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:59:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:59:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 08:59:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:59:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:59:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 08:59:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:59:27,593 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:59:28,240 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:59:28,631 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:28,896 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:29,063 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:29,183 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:29,208 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:29,264 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:59:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:59:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:44.351216+00:00 (in 14.999569 seconds) quotatotalworker stdout | 2025-11-04 08:59:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:44 GMT)" (scheduled at 2025-11-04 08:59:29.351216+00:00) quotatotalworker stdout | 2025-11-04 08:59:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:59:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:59:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:59:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:59:30,697 [285] [DEBUG] [app] Starting request: urn:request:d6863e67-590b-4d47-9774-9348669156a0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:30,701 [292] [DEBUG] [app] Starting request: urn:request:f4ecd245-4a6c-4121-8b89-cf58d351b08e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:30,701 [292] [DEBUG] [app] Ending request: urn:request:f4ecd245-4a6c-4121-8b89-cf58d351b08e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f4ecd245-4a6c-4121-8b89-cf58d351b08e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:59:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:59:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:30,706 [288] [DEBUG] [app] Starting request: urn:request:d4bf1c2a-e086-4fbc-a0ad-0614a37822c5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:30,706 [288] [DEBUG] [app] Ending request: urn:request:d4bf1c2a-e086-4fbc-a0ad-0614a37822c5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d4bf1c2a-e086-4fbc-a0ad-0614a37822c5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:59:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:59:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 08:59:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:30,730 [285] [DEBUG] [app] Ending request: urn:request:d6863e67-590b-4d47-9774-9348669156a0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d6863e67-590b-4d47-9774-9348669156a0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:30,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:59:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:59:30,784 [285] [DEBUG] [app] Starting request: urn:request:f3fe9dfb-5acd-4710-91dd-27c2ab5e6881 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:30,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:30,785 [299] [DEBUG] [app] Starting request: urn:request:167ac3d7-f98f-4bf3-9e9b-883247d8c477 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:30,786 [299] [DEBUG] [app] Ending request: urn:request:167ac3d7-f98f-4bf3-9e9b-883247d8c477 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:167ac3d7-f98f-4bf3-9e9b-883247d8c477', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:59:30,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 08:59:30,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:30,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:30,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:30,790 [287] [DEBUG] [app] Starting request: urn:request:e2746342-9c97-4024-8b36-17fac9353881 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:30,790 [287] [DEBUG] [app] Ending request: urn:request:e2746342-9c97-4024-8b36-17fac9353881 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e2746342-9c97-4024-8b36-17fac9353881', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:59:30,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:59:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:30,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:30,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:30,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:30,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:30,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:30,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:30,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:30,814 [285] [DEBUG] [app] Ending request: urn:request:f3fe9dfb-5acd-4710-91dd-27c2ab5e6881 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f3fe9dfb-5acd-4710-91dd-27c2ab5e6881', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:30,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:59:30,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.032) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:30,880 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:59:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:40.533243+00:00 (in 9.002695 seconds) notificationworker stdout | 2025-11-04 08:59:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:41 GMT)" (scheduled at 2025-11-04 08:59:31.530064+00:00) notificationworker stdout | 2025-11-04 08:59:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:59:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 31, 530842), True, datetime.datetime(2025, 11, 4, 8, 59, 31, 530842), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:59:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:59:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:41 GMT)" executed successfully gcworker stdout | 2025-11-04 08:59:31,671 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 08:59:31,870 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 08:59:31,872 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:59:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 08:59:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:21.434460+00:00 (in 48.998225 seconds) namespacegcworker stdout | 2025-11-04 08:59:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:32 GMT)" (scheduled at 2025-11-04 08:59:32.435813+00:00) namespacegcworker stdout | 2025-11-04 08:59:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 08:59:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:32 GMT)" executed successfully builder stdout | 2025-11-04 08:59:32,806 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:59:32,820 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:59:32,820 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:59:32,829 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 08:59:33,155 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 08:59:33,167 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 08:59:33,261 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:59:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:59:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:48.625192+00:00 (in 15.001743 seconds) exportactionlogsworker stdout | 2025-11-04 08:59:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:33 GMT)" (scheduled at 2025-11-04 08:59:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 08:59:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 08:59:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 33, 623786), True, datetime.datetime(2025, 11, 4, 8, 59, 33, 623786), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 08:59:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 08:59:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 08:59:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:33 GMT)" executed successfully builder stdout | 2025-11-04 08:59:34,055 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 08:59:35,340 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:59:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:59:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:38.437104+00:00 (in 3.000893 seconds) gcworker stdout | 2025-11-04 08:59:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:05 GMT)" (scheduled at 2025-11-04 08:59:35.435809+00:00) gcworker stdout | 2025-11-04 08:59:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037175436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 08:59:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 08:59:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:59:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:05 GMT)" executed successfully servicekey stdout | 2025-11-04 08:59:35,703 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:59:36,339 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:59:36,732 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:59:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:40.124445+00:00 (in 3.001185 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:47 GMT)" (scheduled at 2025-11-04 08:59:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:59:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 37, 123535), True, datetime.datetime(2025, 11, 4, 8, 59, 37, 123535), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:59:37,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:59:37,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:37,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 08:59:37,511 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,522 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,555 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,560 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,563 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,565 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,630 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,726 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 08:59:37,760 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:59:38,029 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:59:38,030 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 08:59:38,032 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 08:59:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 08:59:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:05.435809+00:00 (in 26.998293 seconds) gcworker stdout | 2025-11-04 08:59:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:08 GMT)" (scheduled at 2025-11-04 08:59:38.437104+00:00) gcworker stdout | 2025-11-04 08:59:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 08:59:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246718449, None, 1, 0]) gcworker stdout | 2025-11-04 08:59:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 08:59:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 08:59:38,903 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 08:59:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 08:59:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:01.922827+00:00 (in 21.997783 seconds) securityscanningnotificationworker stdout | 2025-11-04 08:59:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:39 GMT)" (scheduled at 2025-11-04 08:59:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 08:59:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 08:59:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 08:59:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:47.122849+00:00 (in 6.997995 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:40 GMT)" (scheduled at 2025-11-04 08:59:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 08:59:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:59:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:41.530064+00:00 (in 0.996347 seconds) notificationworker stdout | 2025-11-04 08:59:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:04:40 GMT)" (scheduled at 2025-11-04 08:59:40.533243+00:00) notificationworker stdout | 2025-11-04 08:59:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 59, 40, 533966), 'notification/%']) notificationworker stdout | 2025-11-04 08:59:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 59, 40, 533966), True, datetime.datetime(2025, 11, 4, 8, 59, 40, 533966), 0, 'notification/%']) notificationworker stdout | 2025-11-04 08:59:40,550 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 59, 40, 533966), True, datetime.datetime(2025, 11, 4, 8, 59, 40, 533966), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 8, 59, 40, 533966), 'notification/%']) notificationworker stdout | 2025-11-04 08:59:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:04:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 08:59:40,614 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 08:59:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:51.530064+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 08:59:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:51 GMT)" (scheduled at 2025-11-04 08:59:41.530064+00:00) notificationworker stdout | 2025-11-04 08:59:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:59:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 41, 530745), True, datetime.datetime(2025, 11, 4, 8, 59, 41, 530745), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:59:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:59:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:59:42,114 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:59:42,830 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:59:42,844 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:59:42,844 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:59:42,854 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:59:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:59:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:57.523915+00:00 (in 14.090606 seconds) repositorygcworker stdout | 2025-11-04 08:59:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:43 GMT)" (scheduled at 2025-11-04 08:59:43.432867+00:00) repositorygcworker stdout | 2025-11-04 08:59:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 08:59:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 43, 433677), True, datetime.datetime(2025, 11, 4, 8, 59, 43, 433677), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 08:59:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 08:59:43,447 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 08:59:43,447 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 08:59:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:59:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:59.351216+00:00 (in 14.999547 seconds) quotatotalworker stdout | 2025-11-04 08:59:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:59 GMT)" (scheduled at 2025-11-04 08:59:44.351216+00:00) quotatotalworker stdout | 2025-11-04 08:59:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:59:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:59:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:59:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 08:59:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 08:59:44,611 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 08:59:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:59:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:47.479080+00:00 (in 1.998858 seconds) securityworker stdout | 2025-11-04 08:59:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:15 GMT)" (scheduled at 2025-11-04 08:59:45.479769+00:00) securityworker stdout | 2025-11-04 08:59:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:59:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:59:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:59:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:59:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:59:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:59:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:59:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 54, 45, 484085), 1, 2]) securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 54, 45, 484085), 1, 2]) securityworker stdout | 2025-11-04 08:59:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 08:59:45,696 [287] [DEBUG] [app] Starting request: urn:request:7be9663d-2ee5-4308-8490-d9d883189e42 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:45,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:45,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:45,701 [292] [DEBUG] [app] Starting request: urn:request:d182f048-3192-481e-8825-f82022c5e70a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:45,701 [292] [DEBUG] [app] Ending request: urn:request:d182f048-3192-481e-8825-f82022c5e70a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d182f048-3192-481e-8825-f82022c5e70a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 08:59:45,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:59:45,701 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:45,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:45,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:45,705 [285] [DEBUG] [app] Starting request: urn:request:491c209c-7327-4c60-a7da-89aba5ef86bb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:45,705 [285] [DEBUG] [app] Ending request: urn:request:491c209c-7327-4c60-a7da-89aba5ef86bb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:491c209c-7327-4c60-a7da-89aba5ef86bb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 08:59:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 08:59:45,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:45,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:45,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:45,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:45,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:45,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:45,723 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:45,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:45,730 [287] [DEBUG] [app] Ending request: urn:request:7be9663d-2ee5-4308-8490-d9d883189e42 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7be9663d-2ee5-4308-8490-d9d883189e42', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:45,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:59:45,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 08:59:45,784 [288] [DEBUG] [app] Starting request: urn:request:715a7d11-5de8-4d19-8e23-f05aa7ee96b1 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 08:59:45,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:45,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 08:59:45,788 [292] [DEBUG] [app] Starting request: urn:request:46904faa-d3d7-4940-9fc3-921a0017c475 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 08:59:45,788 [292] [DEBUG] [app] Ending request: urn:request:46904faa-d3d7-4940-9fc3-921a0017c475 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:46904faa-d3d7-4940-9fc3-921a0017c475', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 08:59:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 08:59:45,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:45,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 08:59:45,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 08:59:45,792 [285] [DEBUG] [app] Starting request: urn:request:d7145113-0671-40b5-962c-1655d8ad3aea (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 08:59:45,792 [285] [DEBUG] [app] Ending request: urn:request:d7145113-0671-40b5-962c-1655d8ad3aea (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d7145113-0671-40b5-962c-1655d8ad3aea', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:08:59:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 08:59:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:08:59:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 08:59:45,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 08:59:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 08:59:45,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 08:59:45,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 08:59:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 08:59:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 08:59:45,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 08:59:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 08:59:45,817 [288] [DEBUG] [app] Ending request: urn:request:715a7d11-5de8-4d19-8e23-f05aa7ee96b1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:715a7d11-5de8-4d19-8e23-f05aa7ee96b1', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 08:59:45,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 08:59:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:08:59:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:08:59:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) proxycacheblobworker stdout | 2025-11-04 08:59:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:53.124163+00:00 (in 6.000927 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:57 GMT)" (scheduled at 2025-11-04 08:59:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:59:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 47, 123526), True, datetime.datetime(2025, 11, 4, 8, 59, 47, 123526), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:59:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:59:47,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:47,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 08:59:57 GMT)" executed successfully securityworker stdout | 2025-11-04 08:59:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 08:59:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:15.479769+00:00 (in 28.000169 seconds) securityworker stdout | 2025-11-04 08:59:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:17 GMT)" (scheduled at 2025-11-04 08:59:47.479080+00:00) securityworker stdout | 2025-11-04 08:59:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 08:59:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 08:59:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 08:59:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:59:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 08:59:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:59:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 54, 47, 483134), 1, 2]) securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 08:59:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 08:59:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 54, 47, 483134), 1, 2]) securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 08:59:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 08:59:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 08:59:47,548 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:59:48,203 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 08:59:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 08:59:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:33.622985+00:00 (in 44.997281 seconds) exportactionlogsworker stdout | 2025-11-04 08:59:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:48 GMT)" (scheduled at 2025-11-04 08:59:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 08:59:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 08:59:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:59:49,516 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 08:59:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 08:59:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:21.226294+00:00 (in 29.999577 seconds) autopruneworker stdout | 2025-11-04 08:59:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:21 GMT)" (scheduled at 2025-11-04 08:59:51.226294+00:00) autopruneworker stdout | 2025-11-04 08:59:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246731235, None, 1, 0]) autopruneworker stdout | 2025-11-04 08:59:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 08:59:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 08:59:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:59:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 08:59:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:01.530064+00:00 (in 9.999589 seconds) notificationworker stdout | 2025-11-04 08:59:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:01 GMT)" (scheduled at 2025-11-04 08:59:51.530064+00:00) notificationworker stdout | 2025-11-04 08:59:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 08:59:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 51, 530771), True, datetime.datetime(2025, 11, 4, 8, 59, 51, 530771), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 08:59:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 08:59:51,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 08:59:51,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 08:59:51,798 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 08:59:52,854 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 08:59:52,868 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 08:59:52,868 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 08:59:52,878 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 08:59:57.122849+00:00 (in 3.998265 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:04:53 GMT)" (scheduled at 2025-11-04 08:59:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 8, 59, 53, 124843), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 08:59:53,136 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 59, 53, 124843), True, datetime.datetime(2025, 11, 4, 8, 59, 53, 124843), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 08:59:53,140 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 8, 59, 53, 124843), True, datetime.datetime(2025, 11, 4, 8, 59, 53, 124843), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 8, 59, 53, 124843), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 08:59:53,143 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:53,143 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:04:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 08:59:54,637 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 08:59:56,885 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 08:59:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 08:59:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:07.122849+00:00 (in 9.999628 seconds) proxycacheblobworker stdout | 2025-11-04 08:59:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:07 GMT)" (scheduled at 2025-11-04 08:59:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 08:59:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 08:59:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 8, 59, 57, 123482), True, datetime.datetime(2025, 11, 4, 8, 59, 57, 123482), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 08:59:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:09.133645+00:00 (in 12.001495 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:57 GMT)" (scheduled at 2025-11-04 08:59:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 08:59:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 08:59:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 08:59:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 08:59:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 08:59:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 08:59:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 08:59:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 08:59:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 08:59:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:27.222846+00:00 (in 29.999552 seconds) buildlogsarchiver stdout | 2025-11-04 08:59:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:27 GMT)" (scheduled at 2025-11-04 08:59:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 08:59:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 8, 59, 57, 223537), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:59:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 08:59:57,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 08:59:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 08:59:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 08:59:57,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 08:59:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 08:59:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:43.432867+00:00 (in 45.908509 seconds) repositorygcworker stdout | 2025-11-04 08:59:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:57 GMT)" (scheduled at 2025-11-04 08:59:57.523915+00:00) repositorygcworker stdout | 2025-11-04 08:59:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 08:59:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:00:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 08:59:57,602 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 08:59:58,249 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 08:59:58,639 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:58,907 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:59,074 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:59,197 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:59,221 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 08:59:59,278 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 08:59:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 08:59:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:14.351216+00:00 (in 14.999520 seconds) quotatotalworker stdout | 2025-11-04 08:59:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:14 GMT)" (scheduled at 2025-11-04 08:59:59.351216+00:00) quotatotalworker stdout | 2025-11-04 08:59:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 08:59:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 08:59:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 08:59:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:00:00,696 [285] [DEBUG] [app] Starting request: urn:request:dbec0604-1f81-4df8-a246-0c5a678ab9da (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:00,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:00,699 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:00,700 [292] [DEBUG] [app] Starting request: urn:request:21988f2b-1599-43b9-b5b3-01d37e6c9dd7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:00,701 [292] [DEBUG] [app] Ending request: urn:request:21988f2b-1599-43b9-b5b3-01d37e6c9dd7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:21988f2b-1599-43b9-b5b3-01d37e6c9dd7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:00:00,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:00:00,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:00,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:00,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:00,705 [288] [DEBUG] [app] Starting request: urn:request:598b53cc-4146-4c98-bd5d-71b08d1cc4c0 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:00,705 [288] [DEBUG] [app] Ending request: urn:request:598b53cc-4146-4c98-bd5d-71b08d1cc4c0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:598b53cc-4146-4c98-bd5d-71b08d1cc4c0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:00:00,705 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:00:00,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:00,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:00,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:00,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:00,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:00,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:00,729 [285] [DEBUG] [app] Ending request: urn:request:dbec0604-1f81-4df8-a246-0c5a678ab9da (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:dbec0604-1f81-4df8-a246-0c5a678ab9da', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:00,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:00:00,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:00:00,784 [285] [DEBUG] [app] Starting request: urn:request:33f4c5d5-0643-4514-922b-623058829363 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:00,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:00,786 [292] [DEBUG] [app] Starting request: urn:request:604fb41b-5708-4e39-b7a9-a77b025d5616 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:00,786 [292] [DEBUG] [app] Ending request: urn:request:604fb41b-5708-4e39-b7a9-a77b025d5616 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:604fb41b-5708-4e39-b7a9-a77b025d5616', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:00:00,787 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:00:00,787 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:00,788 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:00,790 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:00,791 [288] [DEBUG] [app] Starting request: urn:request:64312206-ab8c-4215-9fe3-225cb99a7509 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:00,791 [288] [DEBUG] [app] Ending request: urn:request:64312206-ab8c-4215-9fe3-225cb99a7509 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:64312206-ab8c-4215-9fe3-225cb99a7509', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:00:00,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:00:00,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:00,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:00,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:00,792 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:00,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:00,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:00,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:00,816 [285] [DEBUG] [app] Ending request: urn:request:33f4c5d5-0643-4514-922b-623058829363 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:33f4c5d5-0643-4514-922b-623058829363', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:00,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:00:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) manifestsubjectbackfillworker stdout | 2025-11-04 09:00:00,889 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:00:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:00:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:11.530064+00:00 (in 9.999530 seconds) notificationworker stdout | 2025-11-04 09:00:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:11 GMT)" (scheduled at 2025-11-04 09:00:01.530064+00:00) notificationworker stdout | 2025-11-04 09:00:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:00:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 1, 530820), True, datetime.datetime(2025, 11, 4, 9, 0, 1, 530820), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:00:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:00:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:00:01,680 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:00:01,879 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:00:01,880 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:00:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:00:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:39.924652+00:00 (in 38.001457 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:00:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:01 GMT)" (scheduled at 2025-11-04 09:00:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:00:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:00:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 1, 923453), True, datetime.datetime(2025, 11, 4, 9, 0, 1, 923453), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:00:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:00:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:00:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:01 GMT)" executed successfully builder stdout | 2025-11-04 09:00:02,878 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:00:02,892 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:00:02,892 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:00:02,902 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 09:00:03,164 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:00:03,177 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:00:03,270 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:00:04,065 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:00:05,348 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:00:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:00:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:08.437104+00:00 (in 3.000926 seconds) gcworker stdout | 2025-11-04 09:00:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:35 GMT)" (scheduled at 2025-11-04 09:00:05.435809+00:00) gcworker stdout | 2025-11-04 09:00:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037205436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:00:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:00:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:00:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:35 GMT)" executed successfully servicekey stdout | 2025-11-04 09:00:05,712 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:00:06,348 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:00:06,741 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:00:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:00:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:17.122849+00:00 (in 9.999589 seconds) proxycacheblobworker stdout | 2025-11-04 09:00:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:17 GMT)" (scheduled at 2025-11-04 09:00:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:00:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:00:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 7, 123556), True, datetime.datetime(2025, 11, 4, 9, 0, 7, 123556), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:00:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:00:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:00:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:00:07,524 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,534 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,569 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,570 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,574 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,579 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,643 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,740 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:07,774 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:00:08,040 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:00:08,041 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:00:08,044 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:00:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:00:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:35.435809+00:00 (in 26.998235 seconds) gcworker stdout | 2025-11-04 09:00:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:38 GMT)" (scheduled at 2025-11-04 09:00:08.437104+00:00) gcworker stdout | 2025-11-04 09:00:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:00:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246748449, None, 1, 0]) gcworker stdout | 2025-11-04 09:00:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:00:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:00:08,912 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:00:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:00:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:57.131795+00:00 (in 47.997726 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:00:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:09 GMT)" (scheduled at 2025-11-04 09:00:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:00:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:00:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:00:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:00:09,147 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:00:10,623 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:00:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:00:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:15.535095+00:00 (in 4.004635 seconds) notificationworker stdout | 2025-11-04 09:00:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:21 GMT)" (scheduled at 2025-11-04 09:00:11.530064+00:00) notificationworker stdout | 2025-11-04 09:00:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:00:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 11, 530744), True, datetime.datetime(2025, 11, 4, 9, 0, 11, 530744), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:00:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:00:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:00:12,123 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:00:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:00:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:12.222828+00:00 (in 59.999583 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:00:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:12 GMT)" (scheduled at 2025-11-04 09:00:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:00:12,224 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:00:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:00:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:12 GMT)" executed successfully builder stdout | 2025-11-04 09:00:12,902 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:00:12,915 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:00:12,915 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:00:12,925 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:00:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:00:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:29.351216+00:00 (in 14.999594 seconds) quotatotalworker stdout | 2025-11-04 09:00:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:29 GMT)" (scheduled at 2025-11-04 09:00:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:00:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:00:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:00:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:00:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:00:14,620 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:00:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:00:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:17.479080+00:00 (in 1.998824 seconds) securityworker stdout | 2025-11-04 09:00:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:45 GMT)" (scheduled at 2025-11-04 09:00:15.479769+00:00) securityworker stdout | 2025-11-04 09:00:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:00:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:00:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:00:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 55, 15, 483844), 1, 2]) securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 55, 15, 483844), 1, 2]) securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:00:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:00:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:21.530064+00:00 (in 5.994561 seconds) notificationworker stdout | 2025-11-04 09:00:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:15 GMT)" (scheduled at 2025-11-04 09:00:15.535095+00:00) notificationworker stdout | 2025-11-04 09:00:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:00:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:00:15,697 [288] [DEBUG] [app] Starting request: urn:request:9b4fd627-31d4-4b60-ba49-a787bc21aee3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:15,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:15,701 [292] [DEBUG] [app] Starting request: urn:request:c9eaec6b-41d1-481b-94dd-db3615f7cd57 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:15,701 [292] [DEBUG] [app] Ending request: urn:request:c9eaec6b-41d1-481b-94dd-db3615f7cd57 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c9eaec6b-41d1-481b-94dd-db3615f7cd57', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:00:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:00:15,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:15,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:15,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:15,706 [285] [DEBUG] [app] Starting request: urn:request:dd812aac-8aa8-499d-9d28-fb1cb7787745 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:15,706 [285] [DEBUG] [app] Ending request: urn:request:dd812aac-8aa8-499d-9d28-fb1cb7787745 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dd812aac-8aa8-499d-9d28-fb1cb7787745', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:00:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:00:15,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:15,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:15,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:15,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:15,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:15,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:15,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:15,730 [288] [DEBUG] [app] Ending request: urn:request:9b4fd627-31d4-4b60-ba49-a787bc21aee3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9b4fd627-31d4-4b60-ba49-a787bc21aee3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:15,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:00:15,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:00:15,784 [285] [DEBUG] [app] Starting request: urn:request:756c4d53-ff7b-459a-9e88-7b727f6df599 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:15,788 [297] [DEBUG] [app] Starting request: urn:request:45e9b2e8-4086-46f5-8939-896dd8ef5e27 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:15,788 [297] [DEBUG] [app] Ending request: urn:request:45e9b2e8-4086-46f5-8939-896dd8ef5e27 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:45e9b2e8-4086-46f5-8939-896dd8ef5e27', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:00:15,789 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:00:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:15,792 [286] [DEBUG] [app] Starting request: urn:request:e4c7ecb9-bb20-4775-8620-3b36f8c5add4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:15,793 [286] [DEBUG] [app] Ending request: urn:request:e4c7ecb9-bb20-4775-8620-3b36f8c5add4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e4c7ecb9-bb20-4775-8620-3b36f8c5add4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:00:15,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:00:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:15,817 [285] [DEBUG] [app] Ending request: urn:request:756c4d53-ff7b-459a-9e88-7b727f6df599 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:756c4d53-ff7b-459a-9e88-7b727f6df599', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:00:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:00:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:00:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:27.122849+00:00 (in 9.999614 seconds) proxycacheblobworker stdout | 2025-11-04 09:00:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:27 GMT)" (scheduled at 2025-11-04 09:00:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:00:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:00:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 17, 123519), True, datetime.datetime(2025, 11, 4, 9, 0, 17, 123519), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:00:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:00:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:00:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:00:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:00:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:45.479769+00:00 (in 28.000152 seconds) securityworker stdout | 2025-11-04 09:00:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:47 GMT)" (scheduled at 2025-11-04 09:00:17.479080+00:00) securityworker stdout | 2025-11-04 09:00:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:00:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:00:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:00:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:00:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:00:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:00:17,557 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:00:18,212 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:00:19,536 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:00:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:00:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:51.226294+00:00 (in 29.999552 seconds) autopruneworker stdout | 2025-11-04 09:00:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:51 GMT)" (scheduled at 2025-11-04 09:00:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:00:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246761234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:00:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:00:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:00:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:00:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:00:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:32.435813+00:00 (in 11.000988 seconds) namespacegcworker stdout | 2025-11-04 09:00:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:21 GMT)" (scheduled at 2025-11-04 09:00:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:00:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:00:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 21, 435063), True, datetime.datetime(2025, 11, 4, 9, 0, 21, 435063), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:00:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:00:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:00:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:00:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:00:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:31.530064+00:00 (in 9.999557 seconds) notificationworker stdout | 2025-11-04 09:00:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:31 GMT)" (scheduled at 2025-11-04 09:00:21.530064+00:00) notificationworker stdout | 2025-11-04 09:00:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:00:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 21, 530779), True, datetime.datetime(2025, 11, 4, 9, 0, 21, 530779), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:00:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:00:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:21,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:00:21,807 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:00:22,926 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:00:22,940 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:00:22,940 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:00:22,950 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:24,646 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:00:26,893 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:00:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:00:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:37.122849+00:00 (in 9.999621 seconds) proxycacheblobworker stdout | 2025-11-04 09:00:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:37 GMT)" (scheduled at 2025-11-04 09:00:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:00:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:00:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 27, 123531), True, datetime.datetime(2025, 11, 4, 9, 0, 27, 123531), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:00:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:00:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:00:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:00:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:00:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:57.222846+00:00 (in 29.999606 seconds) buildlogsarchiver stdout | 2025-11-04 09:00:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:57 GMT)" (scheduled at 2025-11-04 09:00:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:00:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 0, 27, 223448), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:00:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:00:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:00:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:00:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:00:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:00:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:00:27,611 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:00:28,258 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:00:28,649 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:28,918 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:29,086 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:29,210 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:29,234 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:29,292 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:00:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:00:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:44.351216+00:00 (in 14.999616 seconds) quotatotalworker stdout | 2025-11-04 09:00:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:44 GMT)" (scheduled at 2025-11-04 09:00:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:00:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:00:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:00:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:00:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:00:30,697 [288] [DEBUG] [app] Starting request: urn:request:7db3be8f-3639-4d49-92bd-084d57ff5748 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:30,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:30,701 [292] [DEBUG] [app] Starting request: urn:request:df3b3a3a-9336-442f-9744-d0ba012aaf16 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:30,702 [292] [DEBUG] [app] Ending request: urn:request:df3b3a3a-9336-442f-9744-d0ba012aaf16 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:df3b3a3a-9336-442f-9744-d0ba012aaf16', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:00:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:00:30,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:30,706 [285] [DEBUG] [app] Starting request: urn:request:bf6db3d6-b373-4d66-a7c5-809c59015525 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:30,706 [285] [DEBUG] [app] Ending request: urn:request:bf6db3d6-b373-4d66-a7c5-809c59015525 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bf6db3d6-b373-4d66-a7c5-809c59015525', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:00:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:00:30,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:30,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:30,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:30,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:30,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:30,730 [288] [DEBUG] [app] Ending request: urn:request:7db3be8f-3639-4d49-92bd-084d57ff5748 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7db3be8f-3639-4d49-92bd-084d57ff5748', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:30,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:00:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:00:30,784 [288] [DEBUG] [app] Starting request: urn:request:27af2333-b880-49c3-90a8-893087560764 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:30,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:30,785 [298] [DEBUG] [app] Starting request: urn:request:96f1b883-b26b-4bc3-9913-81d2270623f1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:30,786 [298] [DEBUG] [app] Ending request: urn:request:96f1b883-b26b-4bc3-9913-81d2270623f1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:96f1b883-b26b-4bc3-9913-81d2270623f1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:00:30,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:00:30,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:30,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:30,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:30,790 [285] [DEBUG] [app] Starting request: urn:request:324dc2e3-a2e8-4f29-a337-be8c87a3b9c9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:30,790 [285] [DEBUG] [app] Ending request: urn:request:324dc2e3-a2e8-4f29-a337-be8c87a3b9c9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:324dc2e3-a2e8-4f29-a337-be8c87a3b9c9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:00:30,790 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:00:30,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:30,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:30,790 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:30,790 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:30,797 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:30,797 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:30,807 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:30,810 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:30,813 [288] [DEBUG] [app] Ending request: urn:request:27af2333-b880-49c3-90a8-893087560764 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:27af2333-b880-49c3-90a8-893087560764', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:30,813 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.030 47 0.030) gunicorn-web stdout | 2025-11-04 09:00:30,813 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:00:30,898 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:00:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:00:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:41.530064+00:00 (in 9.999544 seconds) notificationworker stdout | 2025-11-04 09:00:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:41 GMT)" (scheduled at 2025-11-04 09:00:31.530064+00:00) notificationworker stdout | 2025-11-04 09:00:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:00:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 31, 530807), True, datetime.datetime(2025, 11, 4, 9, 0, 31, 530807), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:00:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:00:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:00:31,689 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:00:31,888 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:00:31,889 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:00:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:00:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:21.434460+00:00 (in 48.998260 seconds) namespacegcworker stdout | 2025-11-04 09:00:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:32 GMT)" (scheduled at 2025-11-04 09:00:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:00:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:00:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:32 GMT)" executed successfully builder stdout | 2025-11-04 09:00:32,950 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:00:32,964 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:00:32,964 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:00:32,974 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 09:00:33,173 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:00:33,185 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:00:33,279 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:00:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:00:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:48.625192+00:00 (in 15.001757 seconds) exportactionlogsworker stdout | 2025-11-04 09:00:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:33 GMT)" (scheduled at 2025-11-04 09:00:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:00:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:00:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 33, 623794), True, datetime.datetime(2025, 11, 4, 9, 0, 33, 623794), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:00:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:00:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:00:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:33 GMT)" executed successfully builder stdout | 2025-11-04 09:00:34,074 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:00:35,357 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:00:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:00:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:38.437104+00:00 (in 3.000903 seconds) gcworker stdout | 2025-11-04 09:00:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:05 GMT)" (scheduled at 2025-11-04 09:00:35.435809+00:00) gcworker stdout | 2025-11-04 09:00:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037235436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:00:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:00:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:00:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:05 GMT)" executed successfully servicekey stdout | 2025-11-04 09:00:35,721 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:00:36,358 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:00:36,750 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:00:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:00:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:40.124445+00:00 (in 3.001169 seconds) proxycacheblobworker stdout | 2025-11-04 09:00:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:47 GMT)" (scheduled at 2025-11-04 09:00:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:00:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:00:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 37, 123555), True, datetime.datetime(2025, 11, 4, 9, 0, 37, 123555), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:00:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:00:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:00:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:00:37,535 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,546 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,582 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,585 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,592 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,585 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,656 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,760 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:00:37,787 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:00:38,050 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:00:38,052 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:00:38,078 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:00:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:00:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:05.435809+00:00 (in 26.998243 seconds) gcworker stdout | 2025-11-04 09:00:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:08 GMT)" (scheduled at 2025-11-04 09:00:38.437104+00:00) gcworker stdout | 2025-11-04 09:00:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:00:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246778449, None, 1, 0]) gcworker stdout | 2025-11-04 09:00:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:00:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:00:38,921 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:00:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:00:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:01.922827+00:00 (in 21.997767 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:00:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:39 GMT)" (scheduled at 2025-11-04 09:00:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:00:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:00:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:00:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:00:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:47.122849+00:00 (in 6.997974 seconds) proxycacheblobworker stdout | 2025-11-04 09:00:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:40 GMT)" (scheduled at 2025-11-04 09:00:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:00:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:00:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:00:40,695 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:00:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:00:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:51.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:00:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:51 GMT)" (scheduled at 2025-11-04 09:00:41.530064+00:00) notificationworker stdout | 2025-11-04 09:00:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:00:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 41, 530762), True, datetime.datetime(2025, 11, 4, 9, 0, 41, 530762), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:00:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:00:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:00:42,132 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:00:42,974 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:00:42,988 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:00:42,988 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:00:42,998 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:00:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:00:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:57.523915+00:00 (in 14.090648 seconds) repositorygcworker stdout | 2025-11-04 09:00:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:43 GMT)" (scheduled at 2025-11-04 09:00:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:00:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:00:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 43, 433573), True, datetime.datetime(2025, 11, 4, 9, 0, 43, 433573), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:00:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:00:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:00:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:00:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:00:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:59.351216+00:00 (in 14.999587 seconds) quotatotalworker stdout | 2025-11-04 09:00:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:59 GMT)" (scheduled at 2025-11-04 09:00:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:00:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:00:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:00:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:00:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:00:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:00:44,629 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:00:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:00:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:47.479080+00:00 (in 1.998832 seconds) securityworker stdout | 2025-11-04 09:00:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:15 GMT)" (scheduled at 2025-11-04 09:00:45.479769+00:00) securityworker stdout | 2025-11-04 09:00:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:00:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:00:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:00:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 55, 45, 483903), 1, 2]) securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 55, 45, 483903), 1, 2]) securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:00:45,696 [288] [DEBUG] [app] Starting request: urn:request:c9b37917-3d3e-4595-8319-8b49137f621d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:45,701 [297] [DEBUG] [app] Starting request: urn:request:ebc2f2cb-7b2f-4d74-8a69-bbdd8968bb01 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:45,702 [297] [DEBUG] [app] Ending request: urn:request:ebc2f2cb-7b2f-4d74-8a69-bbdd8968bb01 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ebc2f2cb-7b2f-4d74-8a69-bbdd8968bb01', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:00:45,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:00:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:45,706 [285] [DEBUG] [app] Starting request: urn:request:803b42e4-e644-44ce-8e7a-ba160ac6c290 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:45,706 [285] [DEBUG] [app] Ending request: urn:request:803b42e4-e644-44ce-8e7a-ba160ac6c290 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:803b42e4-e644-44ce-8e7a-ba160ac6c290', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:00:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:00:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:45,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:45,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:45,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:45,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:45,730 [288] [DEBUG] [app] Ending request: urn:request:c9b37917-3d3e-4595-8319-8b49137f621d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c9b37917-3d3e-4595-8319-8b49137f621d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:45,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:00:45,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:00:45,783 [285] [DEBUG] [app] Starting request: urn:request:21a1a78b-19d8-429b-ac29-25196cfd9f17 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:00:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:00:45,788 [292] [DEBUG] [app] Starting request: urn:request:8271a456-58cf-42fc-928f-779335f3b29f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:00:45,788 [292] [DEBUG] [app] Ending request: urn:request:8271a456-58cf-42fc-928f-779335f3b29f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8271a456-58cf-42fc-928f-779335f3b29f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:00:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:00:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:00:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:00:45,792 [286] [DEBUG] [app] Starting request: urn:request:b28465ab-3383-4e52-b583-4165c7c77965 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:00:45,792 [286] [DEBUG] [app] Ending request: urn:request:b28465ab-3383-4e52-b583-4165c7c77965 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b28465ab-3383-4e52-b583-4165c7c77965', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:00:45,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:00:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:00:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:00:45,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:00:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:00:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:00:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:00:45,802 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:00:45,802 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:00:45,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:00:45,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:00:45,817 [285] [DEBUG] [app] Ending request: urn:request:21a1a78b-19d8-429b-ac29-25196cfd9f17 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:21a1a78b-19d8-429b-ac29-25196cfd9f17', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:00:45,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:00:45,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:00:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:00:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:00:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:00:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:00:57.122849+00:00 (in 9.999574 seconds) proxycacheblobworker stdout | 2025-11-04 09:00:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:57 GMT)" (scheduled at 2025-11-04 09:00:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:00:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:00:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 47, 123589), True, datetime.datetime(2025, 11, 4, 9, 0, 47, 123589), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:00:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:00:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:00:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:00:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:00:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:00:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:15.479769+00:00 (in 28.000161 seconds) securityworker stdout | 2025-11-04 09:00:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:17 GMT)" (scheduled at 2025-11-04 09:00:47.479080+00:00) securityworker stdout | 2025-11-04 09:00:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:00:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:00:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:00:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:00:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:00:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:00:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 55, 47, 483401), 1, 2]) securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:00:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 55, 47, 483401), 1, 2]) securityworker stdout | 2025-11-04 09:00:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:00:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:00:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:00:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:00:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:00:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:00:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:00:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:00:47,577 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:00:48,221 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:00:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:00:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:33.622985+00:00 (in 44.997292 seconds) exportactionlogsworker stdout | 2025-11-04 09:00:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:48 GMT)" (scheduled at 2025-11-04 09:00:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:00:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:00:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:00:49,545 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:00:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:00:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:21.226294+00:00 (in 29.999525 seconds) autopruneworker stdout | 2025-11-04 09:00:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:21 GMT)" (scheduled at 2025-11-04 09:00:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:00:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246791234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:00:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:00:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:00:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:00:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:00:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:01.530064+00:00 (in 9.999610 seconds) notificationworker stdout | 2025-11-04 09:00:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:01 GMT)" (scheduled at 2025-11-04 09:00:51.530064+00:00) notificationworker stdout | 2025-11-04 09:00:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:00:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 51, 530962), True, datetime.datetime(2025, 11, 4, 9, 0, 51, 530962), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:00:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:00:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:00:51,816 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:00:52,998 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:00:53,013 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:00:53,013 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:00:53,024 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:00:54,656 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:00:56,902 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:00:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:00:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:07.122849+00:00 (in 9.999600 seconds) proxycacheblobworker stdout | 2025-11-04 09:00:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:07 GMT)" (scheduled at 2025-11-04 09:00:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:00:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:00:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 0, 57, 123532), True, datetime.datetime(2025, 11, 4, 9, 0, 57, 123532), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:00:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:00:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:09.133645+00:00 (in 12.001438 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:00:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:57 GMT)" (scheduled at 2025-11-04 09:00:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:00:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:00:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:00:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:00:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:00:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:00:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:00:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:00:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:00:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:27.222846+00:00 (in 29.999570 seconds) buildlogsarchiver stdout | 2025-11-04 09:00:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:27 GMT)" (scheduled at 2025-11-04 09:00:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:00:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 0, 57, 223516), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:00:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:00:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:00:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:00:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:00:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:00:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:00:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:43.432867+00:00 (in 45.908510 seconds) repositorygcworker stdout | 2025-11-04 09:00:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:57 GMT)" (scheduled at 2025-11-04 09:00:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:00:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:00:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:01:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:00:57,620 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:00:58,267 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:00:58,658 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:58,929 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:59,103 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:59,224 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:59,248 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:00:59,306 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:00:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:00:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:14.351216+00:00 (in 14.999559 seconds) quotatotalworker stdout | 2025-11-04 09:00:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:14 GMT)" (scheduled at 2025-11-04 09:00:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:00:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:00:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:00:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:00:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:01:00,697 [285] [DEBUG] [app] Starting request: urn:request:3852c318-c4b8-4fbf-a6cd-12977ce1a227 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:00,701 [292] [DEBUG] [app] Starting request: urn:request:0ff8497a-6885-4a7f-9467-61a4d166a96e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:00,702 [292] [DEBUG] [app] Ending request: urn:request:0ff8497a-6885-4a7f-9467-61a4d166a96e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0ff8497a-6885-4a7f-9467-61a4d166a96e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:01:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:01:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:00,706 [288] [DEBUG] [app] Starting request: urn:request:a5c4f58e-65c8-4e4d-aee0-89fc9daf3576 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:00,706 [288] [DEBUG] [app] Ending request: urn:request:a5c4f58e-65c8-4e4d-aee0-89fc9daf3576 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a5c4f58e-65c8-4e4d-aee0-89fc9daf3576', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:01:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:00,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:00,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:00,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:00,731 [285] [DEBUG] [app] Ending request: urn:request:3852c318-c4b8-4fbf-a6cd-12977ce1a227 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3852c318-c4b8-4fbf-a6cd-12977ce1a227', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:01:00,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:01:00,784 [285] [DEBUG] [app] Starting request: urn:request:d46ef6ac-6be8-465f-b579-bb9fb5b292b1 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:00,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:00,785 [299] [DEBUG] [app] Starting request: urn:request:ed343cb0-2c56-4104-b6a4-f63e05522e1e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:00,786 [299] [DEBUG] [app] Ending request: urn:request:ed343cb0-2c56-4104-b6a4-f63e05522e1e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ed343cb0-2c56-4104-b6a4-f63e05522e1e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:01:00,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:01:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:00,790 [287] [DEBUG] [app] Starting request: urn:request:d708b640-9ca2-4331-ba6a-1c7f7ec7bc92 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:00,790 [287] [DEBUG] [app] Ending request: urn:request:d708b640-9ca2-4331-ba6a-1c7f7ec7bc92 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d708b640-9ca2-4331-ba6a-1c7f7ec7bc92', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:00,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:01:00,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:00,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:00,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:00,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:00,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:00,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:00,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:00,814 [285] [DEBUG] [app] Ending request: urn:request:d46ef6ac-6be8-465f-b579-bb9fb5b292b1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d46ef6ac-6be8-465f-b579-bb9fb5b292b1', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:00,814 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) gunicorn-web stdout | 2025-11-04 09:01:00,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:01:00,907 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:01:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:01:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:11.530064+00:00 (in 9.999601 seconds) notificationworker stdout | 2025-11-04 09:01:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:11 GMT)" (scheduled at 2025-11-04 09:01:01.530064+00:00) notificationworker stdout | 2025-11-04 09:01:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:01:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 1, 530715), True, datetime.datetime(2025, 11, 4, 9, 1, 1, 530715), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:01:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:01:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:01:01,697 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:01:01,897 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:01:01,898 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:01:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:01:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:39.924652+00:00 (in 38.001428 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:01:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:01 GMT)" (scheduled at 2025-11-04 09:01:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:01:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:01:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 1, 923470), True, datetime.datetime(2025, 11, 4, 9, 1, 1, 923470), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:01:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:01:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:01:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:01 GMT)" executed successfully builder stdout | 2025-11-04 09:01:03,024 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:01:03,037 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:01:03,037 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:01:03,047 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 09:01:03,182 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:01:03,194 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:01:03,299 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:01:04,084 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:01:05,366 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:01:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:01:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:08.437104+00:00 (in 3.000879 seconds) gcworker stdout | 2025-11-04 09:01:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:35 GMT)" (scheduled at 2025-11-04 09:01:05.435809+00:00) gcworker stdout | 2025-11-04 09:01:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037265436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:01:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:01:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:01:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:35 GMT)" executed successfully servicekey stdout | 2025-11-04 09:01:05,730 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:01:06,367 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:01:06,759 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:01:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:01:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:17.122849+00:00 (in 9.999588 seconds) proxycacheblobworker stdout | 2025-11-04 09:01:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:17 GMT)" (scheduled at 2025-11-04 09:01:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:01:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:01:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 7, 123529), True, datetime.datetime(2025, 11, 4, 9, 1, 7, 123529), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:01:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:01:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:01:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:01:07,545 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,556 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,592 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,599 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,606 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,614 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,667 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,773 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:07,797 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:01:08,060 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:01:08,062 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:01:08,089 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:01:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:01:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:35.435809+00:00 (in 26.998296 seconds) gcworker stdout | 2025-11-04 09:01:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:38 GMT)" (scheduled at 2025-11-04 09:01:08.437104+00:00) gcworker stdout | 2025-11-04 09:01:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:01:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246808449, None, 1, 0]) gcworker stdout | 2025-11-04 09:01:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:01:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:01:08,930 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:01:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:01:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:57.131795+00:00 (in 47.997741 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:01:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:09 GMT)" (scheduled at 2025-11-04 09:01:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:01:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:01:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:01:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:01:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:01:10,703 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:01:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:01:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:15.535095+00:00 (in 4.004587 seconds) notificationworker stdout | 2025-11-04 09:01:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:21 GMT)" (scheduled at 2025-11-04 09:01:11.530064+00:00) notificationworker stdout | 2025-11-04 09:01:11,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:01:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 11, 531133), True, datetime.datetime(2025, 11, 4, 9, 1, 11, 531133), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:01:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:01:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:01:12,141 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:01:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:01:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:12.222828+00:00 (in 59.999580 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:01:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:12 GMT)" (scheduled at 2025-11-04 09:01:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:01:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:01:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:01:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:12 GMT)" executed successfully builder stdout | 2025-11-04 09:01:13,048 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:01:13,061 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:01:13,061 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:01:13,071 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:01:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:01:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:29.351216+00:00 (in 14.999572 seconds) quotatotalworker stdout | 2025-11-04 09:01:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:29 GMT)" (scheduled at 2025-11-04 09:01:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:01:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:01:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:01:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:01:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:01:14,638 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:01:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:01:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:17.479080+00:00 (in 1.998798 seconds) securityworker stdout | 2025-11-04 09:01:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:45 GMT)" (scheduled at 2025-11-04 09:01:15.479769+00:00) securityworker stdout | 2025-11-04 09:01:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:01:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:01:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:01:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:01:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:01:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 56, 15, 483987), 1, 2]) securityworker stdout | 2025-11-04 09:01:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:01:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 56, 15, 483987), 1, 2]) securityworker stdout | 2025-11-04 09:01:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:01:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:01:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:21.530064+00:00 (in 5.994509 seconds) notificationworker stdout | 2025-11-04 09:01:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:15 GMT)" (scheduled at 2025-11-04 09:01:15.535095+00:00) notificationworker stdout | 2025-11-04 09:01:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:01:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:01:15,696 [285] [DEBUG] [app] Starting request: urn:request:ec73a8c2-5586-43ef-90f7-9c9355763c96 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:15,701 [299] [DEBUG] [app] Starting request: urn:request:e88c7a79-e107-4ab9-9779-2301a4e02e23 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:15,701 [299] [DEBUG] [app] Ending request: urn:request:e88c7a79-e107-4ab9-9779-2301a4e02e23 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e88c7a79-e107-4ab9-9779-2301a4e02e23', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:01:15,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:01:15,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:15,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:15,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:15,705 [288] [DEBUG] [app] Starting request: urn:request:0631e459-f25b-45b7-91c4-5e8bff2cbe66 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:15,705 [288] [DEBUG] [app] Ending request: urn:request:0631e459-f25b-45b7-91c4-5e8bff2cbe66 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0631e459-f25b-45b7-91c4-5e8bff2cbe66', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:01:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:15,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:15,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:15,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:15,730 [285] [DEBUG] [app] Ending request: urn:request:ec73a8c2-5586-43ef-90f7-9c9355763c96 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ec73a8c2-5586-43ef-90f7-9c9355763c96', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:15,730 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:01:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:01:15,784 [285] [DEBUG] [app] Starting request: urn:request:a9c39e8f-7f07-4caa-b458-a8f504c58bda (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:15,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:15,785 [292] [DEBUG] [app] Starting request: urn:request:f1bdc887-0592-474f-a138-e631fc63f7c3 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:15,786 [292] [DEBUG] [app] Ending request: urn:request:f1bdc887-0592-474f-a138-e631fc63f7c3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f1bdc887-0592-474f-a138-e631fc63f7c3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:01:15,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:01:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:15,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:15,790 [288] [DEBUG] [app] Starting request: urn:request:bf879b62-c67b-48a1-929e-78e844b22c07 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:15,790 [288] [DEBUG] [app] Ending request: urn:request:bf879b62-c67b-48a1-929e-78e844b22c07 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bf879b62-c67b-48a1-929e-78e844b22c07', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:01:15,791 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:15,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:15,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:15,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:15,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:15,812 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:15,815 [285] [DEBUG] [app] Ending request: urn:request:a9c39e8f-7f07-4caa-b458-a8f504c58bda (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a9c39e8f-7f07-4caa-b458-a8f504c58bda', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:15,815 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) gunicorn-web stdout | 2025-11-04 09:01:15,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:01:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:01:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:27.122849+00:00 (in 9.999624 seconds) proxycacheblobworker stdout | 2025-11-04 09:01:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:27 GMT)" (scheduled at 2025-11-04 09:01:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:01:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:01:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 17, 123471), True, datetime.datetime(2025, 11, 4, 9, 1, 17, 123471), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:01:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:01:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:01:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:01:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:01:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:45.479769+00:00 (in 28.000137 seconds) securityworker stdout | 2025-11-04 09:01:17,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:47 GMT)" (scheduled at 2025-11-04 09:01:17.479080+00:00) securityworker stdout | 2025-11-04 09:01:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:01:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:01:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:01:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:01:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:01:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:01:17,586 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:01:18,230 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:01:19,554 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:19,624 [288] [DEBUG] [app] Starting request: urn:request:28d4a2b5-ce1d-4739-b855-b33b9e7a3e78 (/api/v1/user/notifications) {'X-Forwarded-For': '10.131.0.6'} gunicorn-web stdout | 2025-11-04 09:01:19,624 [288] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:01:19,624 [288] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:01:19,625 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:01:19,636 [288] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:01:19,637 [288] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:01:19,637 [288] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:01:19,637 [288] [DEBUG] [endpoints.api] Checking permission for user whuquay gunicorn-web stdout | 2025-11-04 09:01:19,637 [288] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:01:19,637 [288] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:01:19,637 [288] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:01:19,637 [288] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:01:19,638 [288] [DEBUG] [peewee] ('(SELECT "t1"."id", "t1"."uuid", "t1"."kind_id", "t1"."metadata_json", "t1"."dismissed", "t1"."lookup_path", "t1"."created", "t1"."created" AS "cd", "t1"."target_id" FROM "notification" AS "t1" INNER JOIN "notificationkind" AS "t2" ON ("t1"."kind_id" = "t2"."id") WHERE (("t1"."dismissed" = %s) AND ("t1"."target_id" = %s))) UNION (SELECT "t3"."id", "t3"."uuid", "t3"."kind_id", "t3"."metadata_json", "t3"."dismissed", "t3"."lookup_path", "t3"."created", "t3"."created" AS "cd", "t3"."target_id" FROM "notification" AS "t3" INNER JOIN "notificationkind" AS "t4" ON ("t3"."kind_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t5"."id" = "t3"."target_id") INNER JOIN "team" AS "t6" ON ("t5"."id" = "t6"."organization_id") INNER JOIN "teamrole" AS "t7" ON ("t6"."role_id" = "t7"."id") INNER JOIN "teammember" AS "t8" ON ("t6"."id" = "t8"."team_id") INNER JOIN "user" AS "t9" ON ("t8"."user_id" = "t9"."id") WHERE (("t3"."dismissed" = %s) AND (("t9"."id" = %s) AND ("t7"."name" = %s)))) ORDER BY cd desc LIMIT %s', [False, 1, False, 1, 'admin', 6]) gunicorn-web stdout | 2025-11-04 09:01:19,644 [288] [DEBUG] [app] Ending request: urn:request:28d4a2b5-ce1d-4739-b855-b33b9e7a3e78 (/api/v1/user/notifications) {'endpoint': 'api.usernotificationlist', 'request_id': 'urn:request:28d4a2b5-ce1d-4739-b855-b33b9e7a3e78', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/user/notifications', 'path': '/api/v1/user/notifications', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:01:19,644 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:01:19,644 [288] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:01:19 +0000] "GET /api/v1/user/notifications HTTP/1.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:01:19 +0000] "GET /api/v1/user/notifications HTTP/2.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.021 957 0.022) autopruneworker stdout | 2025-11-04 09:01:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:01:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:51.226294+00:00 (in 29.999532 seconds) autopruneworker stdout | 2025-11-04 09:01:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:51 GMT)" (scheduled at 2025-11-04 09:01:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:01:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246821234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:01:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:01:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:01:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:01:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:01:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:32.435813+00:00 (in 11.000971 seconds) namespacegcworker stdout | 2025-11-04 09:01:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:21 GMT)" (scheduled at 2025-11-04 09:01:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:01:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:01:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 21, 435184), True, datetime.datetime(2025, 11, 4, 9, 1, 21, 435184), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:01:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:01:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:01:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:01:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:01:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:31.530064+00:00 (in 9.999609 seconds) notificationworker stdout | 2025-11-04 09:01:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:31 GMT)" (scheduled at 2025-11-04 09:01:21.530064+00:00) notificationworker stdout | 2025-11-04 09:01:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:01:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 21, 530777), True, datetime.datetime(2025, 11, 4, 9, 1, 21, 530777), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:01:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:01:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:01:21,825 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:01:23,071 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:01:23,085 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:01:23,085 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:01:23,095 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:24,667 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:01:26,912 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:01:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:01:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:37.122849+00:00 (in 9.999617 seconds) proxycacheblobworker stdout | 2025-11-04 09:01:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:37 GMT)" (scheduled at 2025-11-04 09:01:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:01:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:01:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 27, 123542), True, datetime.datetime(2025, 11, 4, 9, 1, 27, 123542), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:01:27,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:01:27,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:01:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:01:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:01:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:57.222846+00:00 (in 29.999596 seconds) buildlogsarchiver stdout | 2025-11-04 09:01:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:57 GMT)" (scheduled at 2025-11-04 09:01:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:01:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 1, 27, 223447), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:01:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:01:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:01:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:01:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:01:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:01:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:01:27,629 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:01:28,276 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:01:28,666 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:28,939 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:29,114 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:29,238 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:29,261 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:29,320 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:01:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:01:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:44.351216+00:00 (in 14.999587 seconds) quotatotalworker stdout | 2025-11-04 09:01:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:44 GMT)" (scheduled at 2025-11-04 09:01:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:01:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:01:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:01:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:01:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:01:30,697 [287] [DEBUG] [app] Starting request: urn:request:1a86245b-0a2d-443b-bbd0-a3a5bdabf13a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:30,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:30,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:30,701 [292] [DEBUG] [app] Starting request: urn:request:ae519515-f2c1-489c-be50-b2b5d2b31c63 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:30,702 [292] [DEBUG] [app] Ending request: urn:request:ae519515-f2c1-489c-be50-b2b5d2b31c63 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ae519515-f2c1-489c-be50-b2b5d2b31c63', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:01:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:01:30,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:30,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:30,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:30,706 [285] [DEBUG] [app] Starting request: urn:request:6063628d-0725-4c9f-a895-3c73721df5da (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:30,706 [285] [DEBUG] [app] Ending request: urn:request:6063628d-0725-4c9f-a895-3c73721df5da (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6063628d-0725-4c9f-a895-3c73721df5da', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:01:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:01:30,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:30,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:30,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:30,715 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:30,715 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:30,725 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:30,728 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:30,731 [287] [DEBUG] [app] Ending request: urn:request:1a86245b-0a2d-443b-bbd0-a3a5bdabf13a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1a86245b-0a2d-443b-bbd0-a3a5bdabf13a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:30,732 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:01:30,732 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:01:30,784 [285] [DEBUG] [app] Starting request: urn:request:871bbc44-1185-4e28-93c1-c860b9ce6156 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:30,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:30,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:30,789 [299] [DEBUG] [app] Starting request: urn:request:10acdfff-960f-4882-9869-86ceb10d2449 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:30,789 [299] [DEBUG] [app] Ending request: urn:request:10acdfff-960f-4882-9869-86ceb10d2449 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:10acdfff-960f-4882-9869-86ceb10d2449', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:01:30,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:01:30,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:30,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:30,793 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:30,793 [288] [DEBUG] [app] Starting request: urn:request:23aa921a-22a0-4ee7-92a7-324ea98b2949 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:30,794 [288] [DEBUG] [app] Ending request: urn:request:23aa921a-22a0-4ee7-92a7-324ea98b2949 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:23aa921a-22a0-4ee7-92a7-324ea98b2949', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:30,794 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:01:30,794 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:30,794 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:30,795 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:30,795 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:30,802 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:30,802 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:30,812 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:30,815 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:30,818 [285] [DEBUG] [app] Ending request: urn:request:871bbc44-1185-4e28-93c1-c860b9ce6156 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:871bbc44-1185-4e28-93c1-c860b9ce6156', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:30,819 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:01:30,819 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 09:01:30,916 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:01:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:01:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:41.530064+00:00 (in 9.999580 seconds) notificationworker stdout | 2025-11-04 09:01:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:41 GMT)" (scheduled at 2025-11-04 09:01:31.530064+00:00) notificationworker stdout | 2025-11-04 09:01:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:01:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 31, 530761), True, datetime.datetime(2025, 11, 4, 9, 1, 31, 530761), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:01:31,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:01:31,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:01:31,706 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:01:31,906 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:01:31,907 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:01:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:01:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:21.434460+00:00 (in 48.998269 seconds) namespacegcworker stdout | 2025-11-04 09:01:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:32 GMT)" (scheduled at 2025-11-04 09:01:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:01:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:01:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:32 GMT)" executed successfully builder stdout | 2025-11-04 09:01:33,095 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:01:33,109 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:01:33,109 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:01:33,119 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 09:01:33,191 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:01:33,203 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:01:33,308 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:01:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:01:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:48.625192+00:00 (in 15.001815 seconds) exportactionlogsworker stdout | 2025-11-04 09:01:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:33 GMT)" (scheduled at 2025-11-04 09:01:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:01:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:01:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 33, 623672), True, datetime.datetime(2025, 11, 4, 9, 1, 33, 623672), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:01:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:01:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:01:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:33 GMT)" executed successfully builder stdout | 2025-11-04 09:01:34,093 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:01:35,387 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:01:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:01:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:38.437104+00:00 (in 3.000939 seconds) gcworker stdout | 2025-11-04 09:01:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:05 GMT)" (scheduled at 2025-11-04 09:01:35.435809+00:00) gcworker stdout | 2025-11-04 09:01:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037295436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:01:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:01:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:01:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:05 GMT)" executed successfully servicekey stdout | 2025-11-04 09:01:35,751 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:01:36,377 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:01:36,768 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:01:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:01:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:40.124445+00:00 (in 3.001200 seconds) proxycacheblobworker stdout | 2025-11-04 09:01:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:47 GMT)" (scheduled at 2025-11-04 09:01:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:01:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:01:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 37, 123535), True, datetime.datetime(2025, 11, 4, 9, 1, 37, 123535), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:01:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:01:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:01:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:01:37,559 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,567 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,603 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,612 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,619 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,628 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,678 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,784 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:01:37,810 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:01:38,071 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:01:38,072 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:01:38,099 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:01:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:01:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:05.435809+00:00 (in 26.998293 seconds) gcworker stdout | 2025-11-04 09:01:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:08 GMT)" (scheduled at 2025-11-04 09:01:38.437104+00:00) gcworker stdout | 2025-11-04 09:01:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:01:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246838448, None, 1, 0]) gcworker stdout | 2025-11-04 09:01:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:01:38,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:01:38,938 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:01:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:01:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:01.922827+00:00 (in 21.997741 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:01:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:39 GMT)" (scheduled at 2025-11-04 09:01:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:01:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:01:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:01:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:01:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:47.122849+00:00 (in 6.998036 seconds) proxycacheblobworker stdout | 2025-11-04 09:01:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:40 GMT)" (scheduled at 2025-11-04 09:01:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:01:40,124 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:01:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:01:40,712 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:01:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:01:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:51.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:01:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:51 GMT)" (scheduled at 2025-11-04 09:01:41.530064+00:00) notificationworker stdout | 2025-11-04 09:01:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:01:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 41, 530755), True, datetime.datetime(2025, 11, 4, 9, 1, 41, 530755), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:01:41,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:01:41,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:01:42,150 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:01:43,119 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:01:43,133 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:01:43,133 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:01:43,143 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:01:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:01:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:57.523915+00:00 (in 14.090637 seconds) repositorygcworker stdout | 2025-11-04 09:01:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:43 GMT)" (scheduled at 2025-11-04 09:01:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:01:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:01:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 43, 433601), True, datetime.datetime(2025, 11, 4, 9, 1, 43, 433601), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:01:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:01:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:01:43,447 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:01:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:01:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:59.351216+00:00 (in 14.999616 seconds) quotatotalworker stdout | 2025-11-04 09:01:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:59 GMT)" (scheduled at 2025-11-04 09:01:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:01:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:01:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:01:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:01:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:01:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:01:44,647 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:01:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:01:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:47.479080+00:00 (in 1.998805 seconds) securityworker stdout | 2025-11-04 09:01:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:15 GMT)" (scheduled at 2025-11-04 09:01:45.479769+00:00) securityworker stdout | 2025-11-04 09:01:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:01:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:01:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:01:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:45,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:45,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:01:45,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:45,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:45,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 56, 45, 483993), 1, 2]) securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:45,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 56, 45, 483993), 1, 2]) securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:01:45,697 [286] [DEBUG] [app] Starting request: urn:request:6dafc33c-97db-4f9f-8ba1-03805b2e2345 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:45,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:45,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:45,701 [299] [DEBUG] [app] Starting request: urn:request:73bcfabf-589f-452e-8f6e-379deb7ee737 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:45,701 [299] [DEBUG] [app] Ending request: urn:request:73bcfabf-589f-452e-8f6e-379deb7ee737 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:73bcfabf-589f-452e-8f6e-379deb7ee737', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:01:45,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:01:45,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:45,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:45,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:45,705 [285] [DEBUG] [app] Starting request: urn:request:19eab9ab-4db8-4cf2-ac04-652fd40d3f93 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:45,706 [285] [DEBUG] [app] Ending request: urn:request:19eab9ab-4db8-4cf2-ac04-652fd40d3f93 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:19eab9ab-4db8-4cf2-ac04-652fd40d3f93', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:01:45,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:01:45,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:45,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:45,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:45,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:45,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:45,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:45,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:45,730 [286] [DEBUG] [app] Ending request: urn:request:6dafc33c-97db-4f9f-8ba1-03805b2e2345 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6dafc33c-97db-4f9f-8ba1-03805b2e2345', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:45,730 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:01:45,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:01:45,783 [288] [DEBUG] [app] Starting request: urn:request:58b5245c-c6f9-423e-bd3b-be68d0517328 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:01:45,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:45,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:01:45,788 [299] [DEBUG] [app] Starting request: urn:request:c22910ca-7c69-4f48-8a11-c717a69292b1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:01:45,788 [299] [DEBUG] [app] Ending request: urn:request:c22910ca-7c69-4f48-8a11-c717a69292b1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c22910ca-7c69-4f48-8a11-c717a69292b1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:01:45,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:01:45,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:45,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:01:45,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:01:45,793 [287] [DEBUG] [app] Starting request: urn:request:753527ef-7bee-4fc8-82ab-cf2c6b31dadb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:01:45,793 [287] [DEBUG] [app] Ending request: urn:request:753527ef-7bee-4fc8-82ab-cf2c6b31dadb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:753527ef-7bee-4fc8-82ab-cf2c6b31dadb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:01:45,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:01:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:01:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:01:45,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:01:45,794 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:01:45,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:01:45,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:01:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:01:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:01:45,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:01:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:01:45,817 [288] [DEBUG] [app] Ending request: urn:request:58b5245c-c6f9-423e-bd3b-be68d0517328 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:58b5245c-c6f9-423e-bd3b-be68d0517328', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:01:45,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:01:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:01:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:01:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:01:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:01:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:57.122849+00:00 (in 9.999612 seconds) proxycacheblobworker stdout | 2025-11-04 09:01:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:57 GMT)" (scheduled at 2025-11-04 09:01:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:01:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:01:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 47, 123483), True, datetime.datetime(2025, 11, 4, 9, 1, 47, 123483), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:01:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:01:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:01:47,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:01:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:01:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:01:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:15.479769+00:00 (in 28.000186 seconds) securityworker stdout | 2025-11-04 09:01:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:17 GMT)" (scheduled at 2025-11-04 09:01:47.479080+00:00) securityworker stdout | 2025-11-04 09:01:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:01:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:01:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:01:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:01:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:01:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:01:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:47,501 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:01:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 56, 47, 482897), 1, 2]) securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:01:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:01:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:01:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 56, 47, 482897), 1, 2]) securityworker stdout | 2025-11-04 09:01:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:01:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:01:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:01:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:01:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:01:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:01:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:01:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:01:47,595 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:01:48,239 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:01:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:01:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:01:58.624878+00:00 (in 9.999254 seconds) exportactionlogsworker stdout | 2025-11-04 09:01:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:48 GMT)" (scheduled at 2025-11-04 09:01:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:01:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:01:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:01:49,563 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:01:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:01:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:21.226294+00:00 (in 29.999546 seconds) autopruneworker stdout | 2025-11-04 09:01:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:21 GMT)" (scheduled at 2025-11-04 09:01:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:01:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246851234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:01:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:01:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:01:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:01:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:01:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:01.530064+00:00 (in 9.999537 seconds) notificationworker stdout | 2025-11-04 09:01:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:01 GMT)" (scheduled at 2025-11-04 09:01:51.530064+00:00) notificationworker stdout | 2025-11-04 09:01:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:01:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 51, 530829), True, datetime.datetime(2025, 11, 4, 9, 1, 51, 530829), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:01:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:01:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:01:51,834 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:01:53,143 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:01:53,158 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:01:53,158 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:01:53,168 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:01:54,698 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:01:56,921 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:01:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:01:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:07.122849+00:00 (in 9.999644 seconds) proxycacheblobworker stdout | 2025-11-04 09:01:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:07 GMT)" (scheduled at 2025-11-04 09:01:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:01:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:01:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 1, 57, 123543), True, datetime.datetime(2025, 11, 4, 9, 1, 57, 123543), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:01:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:01:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:09.133645+00:00 (in 12.001435 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:01:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:57 GMT)" (scheduled at 2025-11-04 09:01:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:01:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:01:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:01:57,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:01:57,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:01:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:01:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:01:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:01:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:01:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:27.222846+00:00 (in 29.999597 seconds) buildlogsarchiver stdout | 2025-11-04 09:01:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:27 GMT)" (scheduled at 2025-11-04 09:01:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:01:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 1, 57, 223483), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:01:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:01:57,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:01:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:01:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:01:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:01:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:01:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:43.432867+00:00 (in 45.908549 seconds) repositorygcworker stdout | 2025-11-04 09:01:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:57 GMT)" (scheduled at 2025-11-04 09:01:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:01:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:01:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:02:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:01:57,638 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:01:58,285 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:01:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:01:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:33.622985+00:00 (in 34.997694 seconds) exportactionlogsworker stdout | 2025-11-04 09:01:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:06:58 GMT)" (scheduled at 2025-11-04 09:01:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:01:58,626 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 1, 58, 625600), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:01:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 1, 58, 625600), True, datetime.datetime(2025, 11, 4, 9, 1, 58, 625600), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:01:58,642 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 1, 58, 625600), True, datetime.datetime(2025, 11, 4, 9, 1, 58, 625600), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 1, 58, 625600), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:01:58,645 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:01:58,645 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:06:58 GMT)" executed successfully teamsyncworker stdout | 2025-11-04 09:01:58,675 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:58,949 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:59,126 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:59,251 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:59,273 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:01:59,334 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:01:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:01:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:14.351216+00:00 (in 14.999574 seconds) quotatotalworker stdout | 2025-11-04 09:01:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:14 GMT)" (scheduled at 2025-11-04 09:01:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:01:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:01:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:01:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:01:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:02:00,697 [285] [DEBUG] [app] Starting request: urn:request:5d99c3e1-2b95-4015-98e6-99d6b65ccae6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:00,702 [299] [DEBUG] [app] Starting request: urn:request:881a527b-870f-478e-ac84-21c264e6d629 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:00,702 [299] [DEBUG] [app] Ending request: urn:request:881a527b-870f-478e-ac84-21c264e6d629 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:881a527b-870f-478e-ac84-21c264e6d629', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:02:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:02:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:00,704 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:00,706 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:00,707 [287] [DEBUG] [app] Starting request: urn:request:85e3b3a0-8730-4225-8dd1-b62a0ebfb797 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:00,707 [287] [DEBUG] [app] Ending request: urn:request:85e3b3a0-8730-4225-8dd1-b62a0ebfb797 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:85e3b3a0-8730-4225-8dd1-b62a0ebfb797', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:02:00,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:02:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:00,708 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:00,708 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:00,708 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:00,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:00,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:00,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:00,729 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:00,732 [285] [DEBUG] [app] Ending request: urn:request:5d99c3e1-2b95-4015-98e6-99d6b65ccae6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5d99c3e1-2b95-4015-98e6-99d6b65ccae6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:00,732 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:00,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) gunicorn-web stdout | 2025-11-04 09:02:00,783 [288] [DEBUG] [app] Starting request: urn:request:1a13e4e3-2961-41a2-874e-05391cedce6e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:00,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:00,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:00,788 [292] [DEBUG] [app] Starting request: urn:request:de4fd3d9-a592-40be-9e78-018c178335ed (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:00,788 [292] [DEBUG] [app] Ending request: urn:request:de4fd3d9-a592-40be-9e78-018c178335ed (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:de4fd3d9-a592-40be-9e78-018c178335ed', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:02:00,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:02:00,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:00,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:00,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:00,792 [286] [DEBUG] [app] Starting request: urn:request:e95e6f87-bf17-4f5e-bdda-fda54aa4679d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:00,792 [286] [DEBUG] [app] Ending request: urn:request:e95e6f87-bf17-4f5e-bdda-fda54aa4679d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e95e6f87-bf17-4f5e-bdda-fda54aa4679d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:02:00,792 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:02:00,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:00,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:00,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:00,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:00,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:00,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:00,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:00,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:00,816 [288] [DEBUG] [app] Ending request: urn:request:1a13e4e3-2961-41a2-874e-05391cedce6e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1a13e4e3-2961-41a2-874e-05391cedce6e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:00,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:00,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:00,925 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:02:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:02:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:11.530064+00:00 (in 9.999547 seconds) notificationworker stdout | 2025-11-04 09:02:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:11 GMT)" (scheduled at 2025-11-04 09:02:01.530064+00:00) notificationworker stdout | 2025-11-04 09:02:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:02:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 1, 530807), True, datetime.datetime(2025, 11, 4, 9, 2, 1, 530807), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:02:01,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:02:01,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:02:01,715 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:02:01,915 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:02:01,915 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:02:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:02:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:21.924304+00:00 (in 20.001075 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:02:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:01 GMT)" (scheduled at 2025-11-04 09:02:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:02:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:02:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 1, 923484), True, datetime.datetime(2025, 11, 4, 9, 2, 1, 923484), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:02:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:02:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:02:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:01 GMT)" executed successfully builder stdout | 2025-11-04 09:02:03,168 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:02:03,181 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:02:03,181 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:02:03,191 [66] [DEBUG] [data.database] Disconnecting from database. chunkcleanupworker stdout | 2025-11-04 09:02:03,199 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:02:03,211 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:02:03,318 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:02:04,103 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:02:05,395 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:02:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:02:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:08.437104+00:00 (in 3.000887 seconds) gcworker stdout | 2025-11-04 09:02:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:35 GMT)" (scheduled at 2025-11-04 09:02:05.435809+00:00) gcworker stdout | 2025-11-04 09:02:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037325436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:02:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:02:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:02:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:35 GMT)" executed successfully servicekey stdout | 2025-11-04 09:02:05,760 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:02:06,387 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:02:06,777 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:02:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:02:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:17.122849+00:00 (in 9.999581 seconds) proxycacheblobworker stdout | 2025-11-04 09:02:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:17 GMT)" (scheduled at 2025-11-04 09:02:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:02:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:02:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 7, 123560), True, datetime.datetime(2025, 11, 4, 9, 2, 7, 123560), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:02:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:02:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:02:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:02:07,573 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,588 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,614 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,625 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,632 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,643 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,688 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,796 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:07,822 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:02:08,081 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:02:08,083 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:02:08,109 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:02:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:02:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:35.435809+00:00 (in 26.998275 seconds) gcworker stdout | 2025-11-04 09:02:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:38 GMT)" (scheduled at 2025-11-04 09:02:08.437104+00:00) gcworker stdout | 2025-11-04 09:02:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:02:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246868449, None, 1, 0]) gcworker stdout | 2025-11-04 09:02:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:02:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:02:08,947 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:02:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:02:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:57.131795+00:00 (in 47.997719 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:09 GMT)" (scheduled at 2025-11-04 09:02:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:02:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:02:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:02:10,721 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:02:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:02:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:15.535095+00:00 (in 4.004614 seconds) notificationworker stdout | 2025-11-04 09:02:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:21 GMT)" (scheduled at 2025-11-04 09:02:11.530064+00:00) notificationworker stdout | 2025-11-04 09:02:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:02:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 11, 530783), True, datetime.datetime(2025, 11, 4, 9, 2, 11, 530783), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:02:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:02:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:02:12,165 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:02:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:02:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:12.222828+00:00 (in 59.999589 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:02:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:12 GMT)" (scheduled at 2025-11-04 09:02:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:02:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:02:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:02:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:12 GMT)" executed successfully builder stdout | 2025-11-04 09:02:13,192 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:02:13,205 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:02:13,205 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:02:13,215 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:02:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:02:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:29.351216+00:00 (in 14.999604 seconds) quotatotalworker stdout | 2025-11-04 09:02:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:29 GMT)" (scheduled at 2025-11-04 09:02:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:02:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:02:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:02:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:02:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:02:14,656 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:02:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:02:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:17.479080+00:00 (in 1.998838 seconds) securityworker stdout | 2025-11-04 09:02:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:45 GMT)" (scheduled at 2025-11-04 09:02:15.479769+00:00) securityworker stdout | 2025-11-04 09:02:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:02:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:02:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:02:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:02:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:15,500 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:15,500 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:15,500 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 57, 15, 483967), 1, 2]) securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 57, 15, 483967), 1, 2]) securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:02:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:02:15,508 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:02:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:02:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:21.530064+00:00 (in 5.994550 seconds) notificationworker stdout | 2025-11-04 09:02:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:15 GMT)" (scheduled at 2025-11-04 09:02:15.535095+00:00) notificationworker stdout | 2025-11-04 09:02:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:02:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:02:15,696 [285] [DEBUG] [app] Starting request: urn:request:4d15e125-6d86-4518-9fc9-c6c0be77ff8c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:15,701 [292] [DEBUG] [app] Starting request: urn:request:ba35bca6-3019-4cac-a1a1-879ecabece57 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:15,701 [292] [DEBUG] [app] Ending request: urn:request:ba35bca6-3019-4cac-a1a1-879ecabece57 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ba35bca6-3019-4cac-a1a1-879ecabece57', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:02:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:02:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:15,706 [288] [DEBUG] [app] Starting request: urn:request:baab9d7d-f12e-4486-8942-a4df34ec4f83 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:15,706 [288] [DEBUG] [app] Ending request: urn:request:baab9d7d-f12e-4486-8942-a4df34ec4f83 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:baab9d7d-f12e-4486-8942-a4df34ec4f83', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:02:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:02:15,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:15,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:15,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:15,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:15,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:15,731 [285] [DEBUG] [app] Ending request: urn:request:4d15e125-6d86-4518-9fc9-c6c0be77ff8c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4d15e125-6d86-4518-9fc9-c6c0be77ff8c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:15,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:15,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.037 47 0.036) gunicorn-web stdout | 2025-11-04 09:02:15,783 [285] [DEBUG] [app] Starting request: urn:request:807267f3-5011-4533-a010-df3f76b847cb (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:15,785 [292] [DEBUG] [app] Starting request: urn:request:c2639810-2d66-4798-84d1-16755a085d5a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:15,785 [292] [DEBUG] [app] Ending request: urn:request:c2639810-2d66-4798-84d1-16755a085d5a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c2639810-2d66-4798-84d1-16755a085d5a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:02:15,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:02:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:15,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:15,790 [288] [DEBUG] [app] Starting request: urn:request:cc848fcc-5c07-4605-a628-c908842f049f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:15,790 [288] [DEBUG] [app] Ending request: urn:request:cc848fcc-5c07-4605-a628-c908842f049f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cc848fcc-5c07-4605-a628-c908842f049f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:02:15,791 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:02:15,791 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:15,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:15,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:15,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:15,799 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:15,799 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:15,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:15,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:15,815 [285] [DEBUG] [app] Ending request: urn:request:807267f3-5011-4533-a010-df3f76b847cb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:807267f3-5011-4533-a010-df3f76b847cb', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:15,815 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:15,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.032) proxycacheblobworker stdout | 2025-11-04 09:02:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:02:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:27.122849+00:00 (in 9.999605 seconds) proxycacheblobworker stdout | 2025-11-04 09:02:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:27 GMT)" (scheduled at 2025-11-04 09:02:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:02:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:02:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 17, 123527), True, datetime.datetime(2025, 11, 4, 9, 2, 17, 123527), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:02:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:02:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:02:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:02:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:02:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:45.479769+00:00 (in 28.000142 seconds) securityworker stdout | 2025-11-04 09:02:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:47 GMT)" (scheduled at 2025-11-04 09:02:17.479080+00:00) securityworker stdout | 2025-11-04 09:02:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:02:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:02:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:02:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:02:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:02:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:02:17,604 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:02:18,258 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:02:19,573 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:02:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:02:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:51.226294+00:00 (in 29.999537 seconds) autopruneworker stdout | 2025-11-04 09:02:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:51 GMT)" (scheduled at 2025-11-04 09:02:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:02:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246881235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:02:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:02:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:02:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:02:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:02:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:32.435813+00:00 (in 11.000981 seconds) namespacegcworker stdout | 2025-11-04 09:02:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:21 GMT)" (scheduled at 2025-11-04 09:02:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:02:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:02:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 21, 435126), True, datetime.datetime(2025, 11, 4, 9, 2, 21, 435126), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:02:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:02:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:02:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:02:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:02:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:31.530064+00:00 (in 9.999632 seconds) notificationworker stdout | 2025-11-04 09:02:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:31 GMT)" (scheduled at 2025-11-04 09:02:21.530064+00:00) notificationworker stdout | 2025-11-04 09:02:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:02:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 21, 530727), True, datetime.datetime(2025, 11, 4, 9, 2, 21, 530727), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:02:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:02:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:02:21,843 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:02:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:02:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:39.924652+00:00 (in 17.999928 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:02:21,924 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:07:21 GMT)" (scheduled at 2025-11-04 09:02:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:02:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 2, 21, 924958), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:02:21,938 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 2, 21, 924958), True, datetime.datetime(2025, 11, 4, 9, 2, 21, 924958), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:02:21,941 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 2, 21, 924958), True, datetime.datetime(2025, 11, 4, 9, 2, 21, 924958), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 2, 21, 924958), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:02:21,944 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:02:21,945 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:07:21 GMT)" executed successfully builder stdout | 2025-11-04 09:02:23,215 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:02:23,229 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:02:23,229 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:02:23,239 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:24,707 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:02:26,930 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:02:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:02:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:37.122849+00:00 (in 9.999621 seconds) proxycacheblobworker stdout | 2025-11-04 09:02:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:37 GMT)" (scheduled at 2025-11-04 09:02:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:02:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:02:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 27, 123533), True, datetime.datetime(2025, 11, 4, 9, 2, 27, 123533), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:02:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:02:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:02:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:02:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:02:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:57.222846+00:00 (in 29.999608 seconds) buildlogsarchiver stdout | 2025-11-04 09:02:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:57 GMT)" (scheduled at 2025-11-04 09:02:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:02:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 2, 27, 223470), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:02:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:02:27,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:02:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:02:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:02:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:02:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:02:27,647 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:02:28,294 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:02:28,685 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:28,973 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:29,138 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:29,265 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:29,286 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:29,347 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:02:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:02:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:44.351216+00:00 (in 14.999557 seconds) quotatotalworker stdout | 2025-11-04 09:02:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:44 GMT)" (scheduled at 2025-11-04 09:02:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:02:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:02:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:02:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:02:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:02:30,696 [285] [DEBUG] [app] Starting request: urn:request:9416d03d-ec52-4915-a415-6df55caf2652 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:30,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:30,700 [299] [DEBUG] [app] Starting request: urn:request:ecc8fdc5-27c9-4e46-9849-f7559f8cc534 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:30,701 [299] [DEBUG] [app] Ending request: urn:request:ecc8fdc5-27c9-4e46-9849-f7559f8cc534 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ecc8fdc5-27c9-4e46-9849-f7559f8cc534', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:02:30,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:02:30,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:30,705 [288] [DEBUG] [app] Starting request: urn:request:56286fab-dd4c-48a9-95ea-2734952d83de (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:30,705 [288] [DEBUG] [app] Ending request: urn:request:56286fab-dd4c-48a9-95ea-2734952d83de (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:56286fab-dd4c-48a9-95ea-2734952d83de', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:02:30,705 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:02:30,705 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:30,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:30,726 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:30,729 [285] [DEBUG] [app] Ending request: urn:request:9416d03d-ec52-4915-a415-6df55caf2652 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9416d03d-ec52-4915-a415-6df55caf2652', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:02:30,783 [288] [DEBUG] [app] Starting request: urn:request:bdf9f9a2-1289-4fb5-b2bc-4f0ef364fde8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:30,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:30,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:30,787 [297] [DEBUG] [app] Starting request: urn:request:77819627-3e56-471e-821a-a51ded3636bd (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:30,788 [297] [DEBUG] [app] Ending request: urn:request:77819627-3e56-471e-821a-a51ded3636bd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:77819627-3e56-471e-821a-a51ded3636bd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:02:30,788 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:02:30,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:30,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:30,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:30,792 [285] [DEBUG] [app] Starting request: urn:request:a6df96be-cf32-4ea8-ab11-bf44bdee1d73 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:30,792 [285] [DEBUG] [app] Ending request: urn:request:a6df96be-cf32-4ea8-ab11-bf44bdee1d73 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a6df96be-cf32-4ea8-ab11-bf44bdee1d73', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:02:30,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:02:30,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:30,792 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:30,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:30,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:30,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:30,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:30,809 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:30,812 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:30,815 [288] [DEBUG] [app] Ending request: urn:request:bdf9f9a2-1289-4fb5-b2bc-4f0ef364fde8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bdf9f9a2-1289-4fb5-b2bc-4f0ef364fde8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:30,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:30,816 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:30,934 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:02:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:02:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:41.530064+00:00 (in 9.999546 seconds) notificationworker stdout | 2025-11-04 09:02:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:41 GMT)" (scheduled at 2025-11-04 09:02:31.530064+00:00) notificationworker stdout | 2025-11-04 09:02:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:02:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 31, 530821), True, datetime.datetime(2025, 11, 4, 9, 2, 31, 530821), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:02:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:02:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:02:31,735 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:02:31,924 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:02:31,924 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:02:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:02:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:08.435432+00:00 (in 35.999217 seconds) namespacegcworker stdout | 2025-11-04 09:02:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:32 GMT)" (scheduled at 2025-11-04 09:02:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:02:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:02:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:02:33,212 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:02:33,220 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:02:33,239 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:02:33,253 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:02:33,253 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:02:33,263 [66] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:02:33,327 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:02:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:02:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:48.625192+00:00 (in 15.001804 seconds) exportactionlogsworker stdout | 2025-11-04 09:02:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:33 GMT)" (scheduled at 2025-11-04 09:02:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:02:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:02:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 33, 623652), True, datetime.datetime(2025, 11, 4, 9, 2, 33, 623652), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:02:33,635 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:02:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:02:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:33 GMT)" executed successfully builder stdout | 2025-11-04 09:02:34,112 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:02:35,404 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:02:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:02:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:38.437104+00:00 (in 3.000850 seconds) gcworker stdout | 2025-11-04 09:02:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:05 GMT)" (scheduled at 2025-11-04 09:02:35.435809+00:00) gcworker stdout | 2025-11-04 09:02:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037355436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:02:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:02:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:02:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:05 GMT)" executed successfully servicekey stdout | 2025-11-04 09:02:35,769 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:02:36,396 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:02:36,785 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:02:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:02:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:40.124445+00:00 (in 3.001167 seconds) proxycacheblobworker stdout | 2025-11-04 09:02:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:47 GMT)" (scheduled at 2025-11-04 09:02:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:02:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:02:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 37, 123624), True, datetime.datetime(2025, 11, 4, 9, 2, 37, 123624), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:02:37,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:02:37,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:02:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:02:37,587 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,603 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,624 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,643 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,660 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,638 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,698 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,812 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:02:37,833 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:02:38,091 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:02:38,093 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:02:38,120 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:02:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:02:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:05.435809+00:00 (in 26.998253 seconds) gcworker stdout | 2025-11-04 09:02:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:08 GMT)" (scheduled at 2025-11-04 09:02:38.437104+00:00) gcworker stdout | 2025-11-04 09:02:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:02:38,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246898449, None, 1, 0]) gcworker stdout | 2025-11-04 09:02:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:02:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:02:38,955 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:02:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:02:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:01.922827+00:00 (in 21.997752 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:02:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:39 GMT)" (scheduled at 2025-11-04 09:02:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:02:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:02:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:02:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:02:40,125 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:47.122849+00:00 (in 6.997844 seconds) proxycacheblobworker stdout | 2025-11-04 09:02:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:40 GMT)" (scheduled at 2025-11-04 09:02:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:02:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:02:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:02:40,730 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:02:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:02:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:51.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:02:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:51 GMT)" (scheduled at 2025-11-04 09:02:41.530064+00:00) notificationworker stdout | 2025-11-04 09:02:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:02:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 41, 530780), True, datetime.datetime(2025, 11, 4, 9, 2, 41, 530780), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:02:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:02:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:02:42,174 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:02:43,263 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:02:43,277 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:02:43,277 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:02:43,287 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:02:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:02:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:57.523915+00:00 (in 14.090606 seconds) repositorygcworker stdout | 2025-11-04 09:02:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:43 GMT)" (scheduled at 2025-11-04 09:02:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:02:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:02:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 43, 433590), True, datetime.datetime(2025, 11, 4, 9, 2, 43, 433590), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:02:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:02:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:02:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:02:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:02:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:59.351216+00:00 (in 14.999603 seconds) quotatotalworker stdout | 2025-11-04 09:02:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:59 GMT)" (scheduled at 2025-11-04 09:02:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:02:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:02:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:02:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:02:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:02:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:02:44,665 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:02:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:02:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:47.479080+00:00 (in 1.998858 seconds) securityworker stdout | 2025-11-04 09:02:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:15 GMT)" (scheduled at 2025-11-04 09:02:45.479769+00:00) securityworker stdout | 2025-11-04 09:02:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:02:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:02:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:02:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:45,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:45,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:45,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:45,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:45,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 57, 45, 483875), 1, 2]) securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:45,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 57, 45, 483875), 1, 2]) securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:02:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:02:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:02:45,697 [287] [DEBUG] [app] Starting request: urn:request:1f4f650c-5bea-4192-b0d0-96a9454adfff (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:45,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:45,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:45,701 [299] [DEBUG] [app] Starting request: urn:request:9bcca345-8a95-43ea-88d1-8fff77ddc6ac (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:45,702 [299] [DEBUG] [app] Ending request: urn:request:9bcca345-8a95-43ea-88d1-8fff77ddc6ac (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9bcca345-8a95-43ea-88d1-8fff77ddc6ac', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:02:45,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:02:45,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:45,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:45,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:45,706 [285] [DEBUG] [app] Starting request: urn:request:45e8b011-cb97-4287-844e-b315181ab18e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:45,706 [285] [DEBUG] [app] Ending request: urn:request:45e8b011-cb97-4287-844e-b315181ab18e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:45e8b011-cb97-4287-844e-b315181ab18e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:02:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:02:45,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:45,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:45,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:45,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:45,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:45,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:45,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:45,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:45,730 [287] [DEBUG] [app] Ending request: urn:request:1f4f650c-5bea-4192-b0d0-96a9454adfff (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1f4f650c-5bea-4192-b0d0-96a9454adfff', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:45,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:45,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:02:45,783 [285] [DEBUG] [app] Starting request: urn:request:fecae1ae-1f43-4aa6-a34f-37ab9942d32a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:02:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:45,786 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:02:45,787 [292] [DEBUG] [app] Starting request: urn:request:f3a24265-c25a-4595-bc6d-2f78d51b91e2 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:02:45,788 [292] [DEBUG] [app] Ending request: urn:request:f3a24265-c25a-4595-bc6d-2f78d51b91e2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f3a24265-c25a-4595-bc6d-2f78d51b91e2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:02:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:02:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:02:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:02:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:02:45,792 [287] [DEBUG] [app] Starting request: urn:request:26d4d26c-5d76-44d6-9fc5-2d4ecb29d69f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:02:45,792 [287] [DEBUG] [app] Ending request: urn:request:26d4d26c-5d76-44d6-9fc5-2d4ecb29d69f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:26d4d26c-5d76-44d6-9fc5-2d4ecb29d69f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:02:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:02:45,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:02:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:02:45,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:02:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:02:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:02:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:02:45,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:02:45,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:02:45,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:02:45,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:02:45,817 [285] [DEBUG] [app] Ending request: urn:request:fecae1ae-1f43-4aa6-a34f-37ab9942d32a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fecae1ae-1f43-4aa6-a34f-37ab9942d32a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:02:45,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:02:45,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:02:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:02:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:02:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:02:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:02:57.122849+00:00 (in 9.999584 seconds) proxycacheblobworker stdout | 2025-11-04 09:02:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:57 GMT)" (scheduled at 2025-11-04 09:02:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:02:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:02:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 47, 123572), True, datetime.datetime(2025, 11, 4, 9, 2, 47, 123572), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:02:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:02:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:02:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:02:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:02:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:02:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:15.479769+00:00 (in 28.000265 seconds) securityworker stdout | 2025-11-04 09:02:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:17 GMT)" (scheduled at 2025-11-04 09:02:47.479080+00:00) securityworker stdout | 2025-11-04 09:02:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:02:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:02:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:02:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:02:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:47,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:47,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:47,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:47,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:47,505 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 57, 47, 483558), 1, 2]) securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:02:47,508 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:02:47,509 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:02:47,509 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 57, 47, 483558), 1, 2]) securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:02:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:02:47,513 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:02:47,513 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:02:47,613 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:02:48,267 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:02:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:02:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:33.622985+00:00 (in 44.997312 seconds) exportactionlogsworker stdout | 2025-11-04 09:02:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:48 GMT)" (scheduled at 2025-11-04 09:02:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:02:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:02:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:02:49,582 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:02:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:02:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:21.226294+00:00 (in 29.999514 seconds) autopruneworker stdout | 2025-11-04 09:02:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:21 GMT)" (scheduled at 2025-11-04 09:02:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:02:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246911235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:02:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:02:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:02:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:02:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:02:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:01.530064+00:00 (in 9.999590 seconds) notificationworker stdout | 2025-11-04 09:02:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:01 GMT)" (scheduled at 2025-11-04 09:02:51.530064+00:00) notificationworker stdout | 2025-11-04 09:02:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:02:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 51, 530789), True, datetime.datetime(2025, 11, 4, 9, 2, 51, 530789), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:02:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:02:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:02:51,852 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:02:53,287 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:02:53,301 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:02:53,301 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:02:53,311 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:02:54,716 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:02:56,939 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:02:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:02:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:07.122849+00:00 (in 9.999576 seconds) proxycacheblobworker stdout | 2025-11-04 09:02:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:07 GMT)" (scheduled at 2025-11-04 09:02:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:02:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:02:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 2, 57, 123600), True, datetime.datetime(2025, 11, 4, 9, 2, 57, 123600), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:02:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:09.133645+00:00 (in 12.001435 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:57 GMT)" (scheduled at 2025-11-04 09:02:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:02:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:02:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:02:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:02:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:02:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:02:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:02:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:02:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:02:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:27.222846+00:00 (in 29.999619 seconds) buildlogsarchiver stdout | 2025-11-04 09:02:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:27 GMT)" (scheduled at 2025-11-04 09:02:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:02:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 2, 57, 223447), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:02:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:02:57,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:02:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:02:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:02:57,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:02:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:02:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:43.432867+00:00 (in 45.908511 seconds) repositorygcworker stdout | 2025-11-04 09:02:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:57 GMT)" (scheduled at 2025-11-04 09:02:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:02:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:02:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:03:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:02:57,656 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:02:58,303 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:02:58,694 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:58,984 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:59,150 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:59,278 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:02:59,299 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:02:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:02:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:14.351216+00:00 (in 14.999558 seconds) quotatotalworker stdout | 2025-11-04 09:02:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:14 GMT)" (scheduled at 2025-11-04 09:02:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:02:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:02:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) gunicorn-web stdout | 2025-11-04 09:02:59,363 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:02:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:02:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:03:00,697 [288] [DEBUG] [app] Starting request: urn:request:b6235b8a-25de-4848-be87-18d6c6934d24 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:00,701 [299] [DEBUG] [app] Starting request: urn:request:61e07dc4-17a1-4420-bfcd-f5d3e42f17c4 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:00,702 [299] [DEBUG] [app] Ending request: urn:request:61e07dc4-17a1-4420-bfcd-f5d3e42f17c4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:61e07dc4-17a1-4420-bfcd-f5d3e42f17c4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:03:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:03:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:00,706 [287] [DEBUG] [app] Starting request: urn:request:331be795-3c06-4a33-88ff-b288f84c4751 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:00,706 [287] [DEBUG] [app] Ending request: urn:request:331be795-3c06-4a33-88ff-b288f84c4751 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:331be795-3c06-4a33-88ff-b288f84c4751', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:03:00,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:00,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:00,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:00,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:00,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:00,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:00,731 [288] [DEBUG] [app] Ending request: urn:request:b6235b8a-25de-4848-be87-18d6c6934d24 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b6235b8a-25de-4848-be87-18d6c6934d24', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:00,731 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:03:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:03:00,783 [288] [DEBUG] [app] Starting request: urn:request:f148285a-38ce-44c4-9af4-0656d84f0137 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:00,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:00,785 [292] [DEBUG] [app] Starting request: urn:request:0e2d4faa-b600-4735-aa6a-4c2007a39600 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:00,786 [292] [DEBUG] [app] Ending request: urn:request:0e2d4faa-b600-4735-aa6a-4c2007a39600 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0e2d4faa-b600-4735-aa6a-4c2007a39600', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:03:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:03:00,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:00,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:00,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:00,790 [287] [DEBUG] [app] Starting request: urn:request:cc1a3f21-d61b-4a30-bc78-894769fcf14e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:00,790 [287] [DEBUG] [app] Ending request: urn:request:cc1a3f21-d61b-4a30-bc78-894769fcf14e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cc1a3f21-d61b-4a30-bc78-894769fcf14e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:00,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:03:00,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:00,791 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:00,791 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:00,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:00,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:00,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:00,808 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:00,811 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:00,814 [288] [DEBUG] [app] Ending request: urn:request:f148285a-38ce-44c4-9af4-0656d84f0137 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f148285a-38ce-44c4-9af4-0656d84f0137', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:00,815 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:03:00,815 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.032) manifestsubjectbackfillworker stdout | 2025-11-04 09:03:00,942 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:03:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:03:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:11.530064+00:00 (in 9.999515 seconds) notificationworker stdout | 2025-11-04 09:03:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:11 GMT)" (scheduled at 2025-11-04 09:03:01.530064+00:00) notificationworker stdout | 2025-11-04 09:03:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:03:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 1, 530820), True, datetime.datetime(2025, 11, 4, 9, 3, 1, 530820), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:03:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:03:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:03:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:03:01,744 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:03:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:03:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:39.924652+00:00 (in 38.001400 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:03:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:01 GMT)" (scheduled at 2025-11-04 09:03:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:03:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:03:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 1, 923548), True, datetime.datetime(2025, 11, 4, 9, 3, 1, 923548), 0, 'secscanv4/%', 50, 1, 0]) blobuploadcleanupworker stdout | 2025-11-04 09:03:01,933 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:03:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:03:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:03:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:01 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:03:01,944 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:03:03,221 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:03:03,229 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:03:03,311 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:03:03,325 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:03:03,325 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:03:03,335 [66] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:03:03,336 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:03:04,121 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:03:05,413 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:03:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:03:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:08.437104+00:00 (in 3.000897 seconds) gcworker stdout | 2025-11-04 09:03:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:35 GMT)" (scheduled at 2025-11-04 09:03:05.435809+00:00) gcworker stdout | 2025-11-04 09:03:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037385436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:03:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:03:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:03:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:35 GMT)" executed successfully servicekey stdout | 2025-11-04 09:03:05,778 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:03:06,406 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:03:06,794 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:03:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:03:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:17.122849+00:00 (in 9.999588 seconds) proxycacheblobworker stdout | 2025-11-04 09:03:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:17 GMT)" (scheduled at 2025-11-04 09:03:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:03:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:03:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 7, 123556), True, datetime.datetime(2025, 11, 4, 9, 3, 7, 123556), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:03:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:03:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:03:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:03:07,600 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,618 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,638 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,653 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,673 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,675 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,711 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,828 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:07,848 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:03:08,102 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:03:08,103 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:03:08,131 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:03:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:03:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:21.434460+00:00 (in 12.998641 seconds) namespacegcworker stdout | 2025-11-04 09:03:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:08:08 GMT)" (scheduled at 2025-11-04 09:03:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:03:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 3, 8, 436056), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:03:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:03:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:35.435809+00:00 (in 26.998288 seconds) gcworker stdout | 2025-11-04 09:03:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:38 GMT)" (scheduled at 2025-11-04 09:03:08.437104+00:00) gcworker stdout | 2025-11-04 09:03:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) namespacegcworker stdout | 2025-11-04 09:03:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 3, 8, 436056), True, datetime.datetime(2025, 11, 4, 9, 3, 8, 436056), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:03:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246928448, None, 1, 0]) namespacegcworker stdout | 2025-11-04 09:03:08,451 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 3, 8, 436056), True, datetime.datetime(2025, 11, 4, 9, 3, 8, 436056), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 3, 8, 436056), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:03:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:03:08,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:03:08,455 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:03:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:08:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:03:08,975 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:03:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:03:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:57.131795+00:00 (in 47.997795 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:03:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:09 GMT)" (scheduled at 2025-11-04 09:03:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:03:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:03:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:03:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:03:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:03:10,739 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:03:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:03:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:15.535095+00:00 (in 4.004619 seconds) notificationworker stdout | 2025-11-04 09:03:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:21 GMT)" (scheduled at 2025-11-04 09:03:11.530064+00:00) notificationworker stdout | 2025-11-04 09:03:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:03:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 11, 530846), True, datetime.datetime(2025, 11, 4, 9, 3, 11, 530846), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:03:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:03:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:03:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:03:12,183 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:03:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:03:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:12.222828+00:00 (in 59.999622 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:03:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:12 GMT)" (scheduled at 2025-11-04 09:03:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:03:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:03:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:03:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:12 GMT)" executed successfully builder stdout | 2025-11-04 09:03:13,335 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:03:13,349 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:03:13,349 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:03:13,359 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:03:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:03:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:29.351216+00:00 (in 14.999585 seconds) quotatotalworker stdout | 2025-11-04 09:03:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:29 GMT)" (scheduled at 2025-11-04 09:03:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:03:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:03:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:03:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:03:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:03:14,675 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:03:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:03:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:17.479080+00:00 (in 1.998877 seconds) securityworker stdout | 2025-11-04 09:03:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:45 GMT)" (scheduled at 2025-11-04 09:03:15.479769+00:00) securityworker stdout | 2025-11-04 09:03:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:03:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:03:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:03:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 58, 15, 484118), 1, 2]) securityworker stdout | 2025-11-04 09:03:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 58, 15, 484118), 1, 2]) securityworker stdout | 2025-11-04 09:03:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:03:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:03:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:21.530064+00:00 (in 5.994528 seconds) notificationworker stdout | 2025-11-04 09:03:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:15 GMT)" (scheduled at 2025-11-04 09:03:15.535095+00:00) notificationworker stdout | 2025-11-04 09:03:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:03:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:03:15,697 [288] [DEBUG] [app] Starting request: urn:request:188ae638-819f-4be1-890d-cdf66ca5dca1 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:15,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:15,701 [298] [DEBUG] [app] Starting request: urn:request:5c2f3c09-f286-4125-bab0-70079badac23 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:15,702 [298] [DEBUG] [app] Ending request: urn:request:5c2f3c09-f286-4125-bab0-70079badac23 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5c2f3c09-f286-4125-bab0-70079badac23', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:03:15,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:03:15,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:15,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:15,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:15,706 [285] [DEBUG] [app] Starting request: urn:request:e96ad3bd-3bca-41d0-a3ea-0862c7afe026 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:15,706 [285] [DEBUG] [app] Ending request: urn:request:e96ad3bd-3bca-41d0-a3ea-0862c7afe026 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e96ad3bd-3bca-41d0-a3ea-0862c7afe026', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:03:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:03:15,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:15,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:15,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:15,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:15,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:15,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:15,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:15,730 [288] [DEBUG] [app] Ending request: urn:request:188ae638-819f-4be1-890d-cdf66ca5dca1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:188ae638-819f-4be1-890d-cdf66ca5dca1', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:15,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:03:15,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:03:15,783 [285] [DEBUG] [app] Starting request: urn:request:8f421916-5123-400a-88f1-12cf317b30fa (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:15,788 [298] [DEBUG] [app] Starting request: urn:request:09caadc3-df67-4633-aafd-2413a48ae83a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:15,788 [298] [DEBUG] [app] Ending request: urn:request:09caadc3-df67-4633-aafd-2413a48ae83a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:09caadc3-df67-4633-aafd-2413a48ae83a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:03:15,788 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:03:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:15,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:15,792 [288] [DEBUG] [app] Starting request: urn:request:9237287a-5462-4592-bf3c-f4f34a058821 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:15,793 [288] [DEBUG] [app] Ending request: urn:request:9237287a-5462-4592-bf3c-f4f34a058821 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9237287a-5462-4592-bf3c-f4f34a058821', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:03:15,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:03:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:15,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:15,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:15,815 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:15,818 [285] [DEBUG] [app] Ending request: urn:request:8f421916-5123-400a-88f1-12cf317b30fa (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8f421916-5123-400a-88f1-12cf317b30fa', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:15,818 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:03:15,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) proxycacheblobworker stdout | 2025-11-04 09:03:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:03:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:27.122849+00:00 (in 9.999626 seconds) proxycacheblobworker stdout | 2025-11-04 09:03:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:27 GMT)" (scheduled at 2025-11-04 09:03:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:03:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:03:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 17, 123469), True, datetime.datetime(2025, 11, 4, 9, 3, 17, 123469), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:03:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:03:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:03:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:03:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:03:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:45.479769+00:00 (in 28.000177 seconds) securityworker stdout | 2025-11-04 09:03:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:47 GMT)" (scheduled at 2025-11-04 09:03:17.479080+00:00) securityworker stdout | 2025-11-04 09:03:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:03:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:03:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:03:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:03:17,496 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:03:17,496 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:03:17,622 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:03:18,277 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:03:19,591 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:03:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:03:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:51.226294+00:00 (in 29.999545 seconds) autopruneworker stdout | 2025-11-04 09:03:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:51 GMT)" (scheduled at 2025-11-04 09:03:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:03:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246941234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:03:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:03:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:03:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:03:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:03:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:32.435813+00:00 (in 11.000924 seconds) namespacegcworker stdout | 2025-11-04 09:03:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:21 GMT)" (scheduled at 2025-11-04 09:03:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:03:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:03:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 21, 435165), True, datetime.datetime(2025, 11, 4, 9, 3, 21, 435165), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:03:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:03:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:03:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:03:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:03:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:31.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:03:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:31 GMT)" (scheduled at 2025-11-04 09:03:21.530064+00:00) notificationworker stdout | 2025-11-04 09:03:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:03:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 21, 530784), True, datetime.datetime(2025, 11, 4, 9, 3, 21, 530784), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:03:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:03:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:03:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:03:21,861 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:03:23,359 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:03:23,374 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:03:23,374 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:03:23,385 [66] [DEBUG] [data.database] Disconnecting from database. expiredappspecifictokenworker stdout | 2025-11-04 09:03:24,225 [70] [DEBUG] [apscheduler.scheduler] Looking for jobs to run expiredappspecifictokenworker stdout | 2025-11-04 09:03:24,225 [70] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 10:03:24.225132+00:00 (in 3599.999582 seconds) expiredappspecifictokenworker stdout | 2025-11-04 09:03:24,225 [70] [INFO] [apscheduler.executors.default] Running job "ExpiredAppSpecificTokenWorker._gc_expired_tokens (trigger: interval[1:00:00], next run at: 2025-11-04 10:03:24 GMT)" (scheduled at 2025-11-04 09:03:24.225132+00:00) expiredappspecifictokenworker stdout | 2025-11-04 09:03:24,225 [70] [DEBUG] [__main__] Garbage collecting expired app specific tokens with window: 1 day, 0:00:00 expiredappspecifictokenworker stdout | 2025-11-04 09:03:24,225 [70] [DEBUG] [peewee] ('DELETE FROM "appspecificauthtoken" WHERE ("appspecificauthtoken"."expiration" < %s)', [datetime.datetime(2025, 11, 3, 9, 3, 24, 225840)]) expiredappspecifictokenworker stdout | 2025-11-04 09:03:24,238 [70] [DEBUG] [data.database] Disconnecting from database. expiredappspecifictokenworker stdout | 2025-11-04 09:03:24,238 [70] [INFO] [apscheduler.executors.default] Job "ExpiredAppSpecificTokenWorker._gc_expired_tokens (trigger: interval[1:00:00], next run at: 2025-11-04 10:03:24 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:03:24,725 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:03:26,948 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:03:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:03:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:37.122849+00:00 (in 9.999593 seconds) proxycacheblobworker stdout | 2025-11-04 09:03:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:37 GMT)" (scheduled at 2025-11-04 09:03:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:03:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:03:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 27, 123520), True, datetime.datetime(2025, 11, 4, 9, 3, 27, 123520), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:03:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:03:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:03:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:03:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:03:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:57.222846+00:00 (in 29.999589 seconds) buildlogsarchiver stdout | 2025-11-04 09:03:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:57 GMT)" (scheduled at 2025-11-04 09:03:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:03:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 3, 27, 223544), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:03:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:03:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:03:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:03:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:03:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:03:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:03:27,664 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:03:28,312 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:03:28,703 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:28,995 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:29,166 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:29,290 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:29,312 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:03:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:03:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:44.351216+00:00 (in 14.999613 seconds) quotatotalworker stdout | 2025-11-04 09:03:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:44 GMT)" (scheduled at 2025-11-04 09:03:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:03:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:03:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:03:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:03:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:03:29,378 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:30,697 [285] [DEBUG] [app] Starting request: urn:request:b2e10012-2ea9-4ba1-abb6-0920b46cbdb1 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:30,701 [298] [DEBUG] [app] Starting request: urn:request:cf3fb895-097c-4e4d-91fb-f625dd6582df (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:30,702 [298] [DEBUG] [app] Ending request: urn:request:cf3fb895-097c-4e4d-91fb-f625dd6582df (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cf3fb895-097c-4e4d-91fb-f625dd6582df', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:03:30,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:03:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:30,706 [288] [DEBUG] [app] Starting request: urn:request:8dcbce35-2b89-43a2-a72d-add4c59acf92 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:30,706 [288] [DEBUG] [app] Ending request: urn:request:8dcbce35-2b89-43a2-a72d-add4c59acf92 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8dcbce35-2b89-43a2-a72d-add4c59acf92', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:03:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:30,730 [285] [DEBUG] [app] Ending request: urn:request:b2e10012-2ea9-4ba1-abb6-0920b46cbdb1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b2e10012-2ea9-4ba1-abb6-0920b46cbdb1', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:03:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:03:30,783 [285] [DEBUG] [app] Starting request: urn:request:cf5179e4-ae01-47a3-95db-8f2c61e5d3cf (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:30,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:30,785 [298] [DEBUG] [app] Starting request: urn:request:722e1f8c-2900-4173-ba0a-db4f38430aab (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:30,785 [298] [DEBUG] [app] Ending request: urn:request:722e1f8c-2900-4173-ba0a-db4f38430aab (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:722e1f8c-2900-4173-ba0a-db4f38430aab', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:03:30,785 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:03:30,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:30,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:30,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:30,789 [287] [DEBUG] [app] Starting request: urn:request:23e82507-e09a-422d-8da8-5e272a7952e8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:30,789 [287] [DEBUG] [app] Ending request: urn:request:23e82507-e09a-422d-8da8-5e272a7952e8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:23e82507-e09a-422d-8da8-5e272a7952e8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:03:30,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:30,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:30,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:30,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:30,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:30,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:30,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:30,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:30,813 [285] [DEBUG] [app] Ending request: urn:request:cf5179e4-ae01-47a3-95db-8f2c61e5d3cf (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:cf5179e4-ae01-47a3-95db-8f2c61e5d3cf', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:30,813 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) gunicorn-web stdout | 2025-11-04 09:03:30,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:03:30,951 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:03:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:03:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:41.530064+00:00 (in 9.999516 seconds) notificationworker stdout | 2025-11-04 09:03:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:41 GMT)" (scheduled at 2025-11-04 09:03:31.530064+00:00) notificationworker stdout | 2025-11-04 09:03:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:03:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 31, 530815), True, datetime.datetime(2025, 11, 4, 9, 3, 31, 530815), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:03:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:03:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:03:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:03:31,752 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:03:31,941 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:03:31,953 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:03:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:03:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:21.434460+00:00 (in 48.998265 seconds) namespacegcworker stdout | 2025-11-04 09:03:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:32 GMT)" (scheduled at 2025-11-04 09:03:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:03:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:03:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:03:33,238 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:03:33,229 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:03:33,345 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:03:33,386 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:03:33,399 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:03:33,399 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:03:33,409 [66] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:03:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:03:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:48.625192+00:00 (in 15.001783 seconds) exportactionlogsworker stdout | 2025-11-04 09:03:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:33 GMT)" (scheduled at 2025-11-04 09:03:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:03:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:03:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 33, 623678), True, datetime.datetime(2025, 11, 4, 9, 3, 33, 623678), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:03:33,635 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:03:33,635 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:03:33,635 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:33 GMT)" executed successfully builder stdout | 2025-11-04 09:03:34,131 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:03:35,422 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:03:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:03:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:38.437104+00:00 (in 3.000869 seconds) gcworker stdout | 2025-11-04 09:03:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:05 GMT)" (scheduled at 2025-11-04 09:03:35.435809+00:00) gcworker stdout | 2025-11-04 09:03:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037415436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:03:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:03:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:03:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:05 GMT)" executed successfully servicekey stdout | 2025-11-04 09:03:35,787 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:03:36,416 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:03:36,803 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:03:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:03:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:40.124445+00:00 (in 3.001218 seconds) proxycacheblobworker stdout | 2025-11-04 09:03:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:47 GMT)" (scheduled at 2025-11-04 09:03:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:03:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:03:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 37, 123510), True, datetime.datetime(2025, 11, 4, 9, 3, 37, 123510), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:03:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:03:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:03:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:03:37,612 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,630 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,652 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,664 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,684 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,685 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,725 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,842 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:03:37,870 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:03:38,113 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:03:38,114 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:03:38,141 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:03:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:03:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:05.435809+00:00 (in 26.998301 seconds) gcworker stdout | 2025-11-04 09:03:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:08 GMT)" (scheduled at 2025-11-04 09:03:38.437104+00:00) gcworker stdout | 2025-11-04 09:03:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:03:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246958449, None, 1, 0]) gcworker stdout | 2025-11-04 09:03:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:03:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:03:38,984 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:03:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:03:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:01.922827+00:00 (in 21.997726 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:03:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:39 GMT)" (scheduled at 2025-11-04 09:03:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:03:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:03:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:03:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:03:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:47.122849+00:00 (in 6.998011 seconds) proxycacheblobworker stdout | 2025-11-04 09:03:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:40 GMT)" (scheduled at 2025-11-04 09:03:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:03:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:03:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:03:40,747 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:03:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:03:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:51.530064+00:00 (in 9.999589 seconds) notificationworker stdout | 2025-11-04 09:03:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:51 GMT)" (scheduled at 2025-11-04 09:03:41.530064+00:00) notificationworker stdout | 2025-11-04 09:03:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:03:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 41, 530749), True, datetime.datetime(2025, 11, 4, 9, 3, 41, 530749), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:03:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:03:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:03:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:03:42,192 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:03:43,410 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:03:43,423 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:03:43,423 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics repositorygcworker stdout | 2025-11-04 09:03:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:03:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:57.523915+00:00 (in 14.090676 seconds) repositorygcworker stdout | 2025-11-04 09:03:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:43 GMT)" (scheduled at 2025-11-04 09:03:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:03:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:03:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 43, 433479), True, datetime.datetime(2025, 11, 4, 9, 3, 43, 433479), 0, 'repositorygc/%', 50, 1, 0]) builder stdout | 2025-11-04 09:03:43,440 [66] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:03:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:03:43,445 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:03:43,445 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:03:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:03:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:59.351216+00:00 (in 14.999585 seconds) quotatotalworker stdout | 2025-11-04 09:03:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:59 GMT)" (scheduled at 2025-11-04 09:03:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:03:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:03:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:03:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:03:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:03:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:03:44,685 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:03:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:03:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:47.479080+00:00 (in 1.998877 seconds) securityworker stdout | 2025-11-04 09:03:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:15 GMT)" (scheduled at 2025-11-04 09:03:45.479769+00:00) securityworker stdout | 2025-11-04 09:03:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:03:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:03:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:03:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:45,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 58, 45, 483600), 1, 2]) securityworker stdout | 2025-11-04 09:03:45,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:45,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:45,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 58, 45, 483600), 1, 2]) securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:45,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:45,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:03:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:03:45,696 [288] [DEBUG] [app] Starting request: urn:request:a65fb9c2-5d0a-48c4-b38e-e8067a8aaadc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:45,701 [298] [DEBUG] [app] Starting request: urn:request:cbcd7fbb-7c17-44ab-b9b8-8d75d90a569d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:45,701 [298] [DEBUG] [app] Ending request: urn:request:cbcd7fbb-7c17-44ab-b9b8-8d75d90a569d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cbcd7fbb-7c17-44ab-b9b8-8d75d90a569d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:03:45,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:03:45,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:45,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:45,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:45,705 [287] [DEBUG] [app] Starting request: urn:request:a336e537-aa31-4bb4-9ab6-14c6e365e3d3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:45,705 [287] [DEBUG] [app] Ending request: urn:request:a336e537-aa31-4bb4-9ab6-14c6e365e3d3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a336e537-aa31-4bb4-9ab6-14c6e365e3d3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:03:45,705 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:45,705 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:45,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:45,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:45,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:45,713 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:45,713 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:45,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:45,726 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:45,729 [288] [DEBUG] [app] Ending request: urn:request:a65fb9c2-5d0a-48c4-b38e-e8067a8aaadc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a65fb9c2-5d0a-48c4-b38e-e8067a8aaadc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:45,729 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:03:45,729 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:03:45,783 [288] [DEBUG] [app] Starting request: urn:request:a0d2c121-6a71-4f83-b342-5a38bc5a2fc1 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:03:45,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:03:45,785 [292] [DEBUG] [app] Starting request: urn:request:cc5b6b6b-4f8c-4039-a675-4badeb68c657 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:03:45,785 [292] [DEBUG] [app] Ending request: urn:request:cc5b6b6b-4f8c-4039-a675-4badeb68c657 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cc5b6b6b-4f8c-4039-a675-4badeb68c657', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:03:45,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:03:45,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:45,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:03:45,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:03:45,789 [287] [DEBUG] [app] Starting request: urn:request:c799631a-b372-40e7-b8dc-e1204a8b22cb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:03:45,790 [287] [DEBUG] [app] Ending request: urn:request:c799631a-b372-40e7-b8dc-e1204a8b22cb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c799631a-b372-40e7-b8dc-e1204a8b22cb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:03:45,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:03:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:03:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:03:45,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:03:45,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:03:45,790 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:03:45,790 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:03:45,797 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:03:45,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:03:45,807 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:03:45,811 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:03:45,814 [288] [DEBUG] [app] Ending request: urn:request:a0d2c121-6a71-4f83-b342-5a38bc5a2fc1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a0d2c121-6a71-4f83-b342-5a38bc5a2fc1', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:03:45,814 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:03:45,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:03:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:03:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:03:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:03:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:03:57.122849+00:00 (in 9.999583 seconds) proxycacheblobworker stdout | 2025-11-04 09:03:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:57 GMT)" (scheduled at 2025-11-04 09:03:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:03:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:03:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 47, 123587), True, datetime.datetime(2025, 11, 4, 9, 3, 47, 123587), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:03:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:03:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:03:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:03:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:03:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:03:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:15.479769+00:00 (in 28.000242 seconds) securityworker stdout | 2025-11-04 09:03:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:17 GMT)" (scheduled at 2025-11-04 09:03:47.479080+00:00) securityworker stdout | 2025-11-04 09:03:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:03:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:03:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:03:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:03:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:03:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 58, 47, 483061), 1, 2]) securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:03:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:03:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 58, 47, 483061), 1, 2]) securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:03:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:03:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:03:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:03:47,631 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:03:48,287 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:03:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:03:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:33.622985+00:00 (in 44.997347 seconds) exportactionlogsworker stdout | 2025-11-04 09:03:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:48 GMT)" (scheduled at 2025-11-04 09:03:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:03:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:03:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:03:49,600 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:03:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:03:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:21.226294+00:00 (in 29.999541 seconds) autopruneworker stdout | 2025-11-04 09:03:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:21 GMT)" (scheduled at 2025-11-04 09:03:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:03:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762246971235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:03:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:03:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:03:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:03:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:03:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:01.530064+00:00 (in 9.999622 seconds) notificationworker stdout | 2025-11-04 09:03:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:01 GMT)" (scheduled at 2025-11-04 09:03:51.530064+00:00) notificationworker stdout | 2025-11-04 09:03:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:03:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 51, 530700), True, datetime.datetime(2025, 11, 4, 9, 3, 51, 530700), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:03:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:03:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:03:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:03:51,870 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:03:53,440 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:03:53,457 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:03:53,457 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:03:53,467 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:03:54,734 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:03:56,958 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:03:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:03:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:07.122849+00:00 (in 9.999580 seconds) proxycacheblobworker stdout | 2025-11-04 09:03:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:07 GMT)" (scheduled at 2025-11-04 09:03:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:03:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:03:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 3, 57, 123532), True, datetime.datetime(2025, 11, 4, 9, 3, 57, 123532), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:03:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:03:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:09.133645+00:00 (in 12.001455 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:03:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:57 GMT)" (scheduled at 2025-11-04 09:03:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:03:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:03:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:03:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:03:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:03:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:03:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:03:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:03:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:03:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:27.222846+00:00 (in 29.999570 seconds) buildlogsarchiver stdout | 2025-11-04 09:03:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:27 GMT)" (scheduled at 2025-11-04 09:03:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:03:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 3, 57, 223530), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:03:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:03:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:03:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:03:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:03:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:03:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:03:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:02.523412+00:00 (in 4.999085 seconds) repositorygcworker stdout | 2025-11-04 09:03:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:57 GMT)" (scheduled at 2025-11-04 09:03:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:03:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:03:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:03:57,685 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:03:58,332 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:03:58,723 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:59,005 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:59,179 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:59,304 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:03:59,325 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:03:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:03:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:14.351216+00:00 (in 14.999536 seconds) quotatotalworker stdout | 2025-11-04 09:03:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:14 GMT)" (scheduled at 2025-11-04 09:03:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:03:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:03:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:03:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:03:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:03:59,392 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:00,697 [285] [DEBUG] [app] Starting request: urn:request:8bf8f8d1-4e70-441b-83ef-01b7bd3203e5 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:00,702 [297] [DEBUG] [app] Starting request: urn:request:12ffa50b-7140-46ad-a33a-5b91e1ee5115 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:00,702 [297] [DEBUG] [app] Ending request: urn:request:12ffa50b-7140-46ad-a33a-5b91e1ee5115 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:12ffa50b-7140-46ad-a33a-5b91e1ee5115', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:00,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:04:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:00,706 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:00,706 [287] [DEBUG] [app] Starting request: urn:request:a2d52c68-374a-49d4-8347-1c907f363b5c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:00,707 [287] [DEBUG] [app] Ending request: urn:request:a2d52c68-374a-49d4-8347-1c907f363b5c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a2d52c68-374a-49d4-8347-1c907f363b5c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:00,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:00,708 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:00,708 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:00,708 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:00,716 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:00,716 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:00,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:00,729 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:00,732 [285] [DEBUG] [app] Ending request: urn:request:8bf8f8d1-4e70-441b-83ef-01b7bd3203e5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8bf8f8d1-4e70-441b-83ef-01b7bd3203e5', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:00,732 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:00,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.037) gunicorn-web stdout | 2025-11-04 09:04:00,784 [286] [DEBUG] [app] Starting request: urn:request:d06f0bb9-ac8d-4470-bc8f-b4c854d5a2fc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:00,784 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:00,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:00,788 [292] [DEBUG] [app] Starting request: urn:request:dccda75d-f054-448d-9257-3c9731ae7400 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:00,788 [292] [DEBUG] [app] Ending request: urn:request:dccda75d-f054-448d-9257-3c9731ae7400 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:dccda75d-f054-448d-9257-3c9731ae7400', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:00,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:04:00,789 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:00,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:00,792 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:00,792 [285] [DEBUG] [app] Starting request: urn:request:a92e712f-ede4-41b6-9d3a-3f16a83d42b0 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:00,793 [285] [DEBUG] [app] Ending request: urn:request:a92e712f-ede4-41b6-9d3a-3f16a83d42b0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a92e712f-ede4-41b6-9d3a-3f16a83d42b0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:00,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:04:00,793 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:00,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:00,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:00,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:00,801 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:00,801 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:00,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:00,813 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:00,816 [286] [DEBUG] [app] Ending request: urn:request:d06f0bb9-ac8d-4470-bc8f-b4c854d5a2fc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d06f0bb9-ac8d-4470-bc8f-b4c854d5a2fc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:00,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:00,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:00,960 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:04:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:11.530064+00:00 (in 9.999590 seconds) notificationworker stdout | 2025-11-04 09:04:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:11 GMT)" (scheduled at 2025-11-04 09:04:01.530064+00:00) notificationworker stdout | 2025-11-04 09:04:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 1, 530770), True, datetime.datetime(2025, 11, 4, 9, 4, 1, 530770), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:04:01,761 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:04:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:04:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:39.924652+00:00 (in 38.001419 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:04:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:01 GMT)" (scheduled at 2025-11-04 09:04:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:04:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:04:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 1, 923516), True, datetime.datetime(2025, 11, 4, 9, 4, 1, 923516), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:04:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:04:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:04:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:01 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:04:01,950 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:04:01,961 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:04:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:43.432867+00:00 (in 40.909135 seconds) repositorygcworker stdout | 2025-11-04 09:04:02,523 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:02 GMT)" (scheduled at 2025-11-04 09:04:02.523412+00:00) repositorygcworker stdout | 2025-11-04 09:04:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 4, 2, 523949), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:04:02,535 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 2, 523949), True, datetime.datetime(2025, 11, 4, 9, 4, 2, 523949), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:04:02,539 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 2, 523949), True, datetime.datetime(2025, 11, 4, 9, 4, 2, 523949), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 4, 2, 523949), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:04:02,542 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:04:02,542 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:02 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:04:03,249 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:04:03,247 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:04:03,354 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:04:03,467 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:04:03,482 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:04:03,482 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:04:03,492 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:04:04,141 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:04:05,431 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:04:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:04:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:08.437104+00:00 (in 3.000938 seconds) gcworker stdout | 2025-11-04 09:04:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:35 GMT)" (scheduled at 2025-11-04 09:04:05.435809+00:00) gcworker stdout | 2025-11-04 09:04:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037445436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:04:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:04:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:04:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:35 GMT)" executed successfully servicekey stdout | 2025-11-04 09:04:05,796 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:04:06,425 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:04:06,812 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:17.122849+00:00 (in 9.999602 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:17 GMT)" (scheduled at 2025-11-04 09:04:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 7, 123527), True, datetime.datetime(2025, 11, 4, 9, 4, 7, 123527), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:07,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:04:07,623 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,643 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,679 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,692 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,696 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,696 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,737 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,856 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:07,884 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:04:08,139 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:04:08,139 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:04:08,153 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:04:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:04:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:35.435809+00:00 (in 26.998261 seconds) gcworker stdout | 2025-11-04 09:04:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:38 GMT)" (scheduled at 2025-11-04 09:04:08.437104+00:00) gcworker stdout | 2025-11-04 09:04:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:04:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246988449, None, 1, 0]) gcworker stdout | 2025-11-04 09:04:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:04:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:04:08,993 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:04:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:04:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:57.131795+00:00 (in 47.997712 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:09 GMT)" (scheduled at 2025-11-04 09:04:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:04:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:04:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:04:10,756 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:04:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:15.535095+00:00 (in 4.004566 seconds) notificationworker stdout | 2025-11-04 09:04:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:21 GMT)" (scheduled at 2025-11-04 09:04:11.530064+00:00) notificationworker stdout | 2025-11-04 09:04:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 11, 530784), True, datetime.datetime(2025, 11, 4, 9, 4, 11, 530784), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:04:12,201 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:04:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:04:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:12.222828+00:00 (in 59.999595 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:04:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:12 GMT)" (scheduled at 2025-11-04 09:04:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:04:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:04:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:04:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:12 GMT)" executed successfully builder stdout | 2025-11-04 09:04:13,492 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:04:13,506 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:04:13,506 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:04:13,516 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:04:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:04:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:29.351216+00:00 (in 14.999546 seconds) quotatotalworker stdout | 2025-11-04 09:04:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:29 GMT)" (scheduled at 2025-11-04 09:04:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:04:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:04:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:04:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:04:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:04:14,694 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:04:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:04:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:17.479080+00:00 (in 1.998844 seconds) securityworker stdout | 2025-11-04 09:04:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:45 GMT)" (scheduled at 2025-11-04 09:04:15.479769+00:00) securityworker stdout | 2025-11-04 09:04:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:04:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:04:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:04:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 15, 483662), 1, 2]) securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 15, 483662), 1, 2]) securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:04:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:04:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:21.530064+00:00 (in 5.994528 seconds) notificationworker stdout | 2025-11-04 09:04:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:15 GMT)" (scheduled at 2025-11-04 09:04:15.535095+00:00) notificationworker stdout | 2025-11-04 09:04:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:04:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:04:15,697 [286] [DEBUG] [app] Starting request: urn:request:c7ac2664-7eb8-4324-af7d-da858548ecfa (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:15,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:15,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:15,701 [292] [DEBUG] [app] Starting request: urn:request:167dd612-0c3b-48cb-8b3b-23ce847b27d0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:15,702 [292] [DEBUG] [app] Ending request: urn:request:167dd612-0c3b-48cb-8b3b-23ce847b27d0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:167dd612-0c3b-48cb-8b3b-23ce847b27d0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:04:15,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:15,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:15,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:15,706 [285] [DEBUG] [app] Starting request: urn:request:1e6363fb-db68-4314-af23-44f940c6eaaa (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:15,706 [285] [DEBUG] [app] Ending request: urn:request:1e6363fb-db68-4314-af23-44f940c6eaaa (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1e6363fb-db68-4314-af23-44f940c6eaaa', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:15,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:15,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:15,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:15,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:15,715 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:15,715 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:15,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:15,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:15,730 [286] [DEBUG] [app] Ending request: urn:request:c7ac2664-7eb8-4324-af7d-da858548ecfa (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c7ac2664-7eb8-4324-af7d-da858548ecfa', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:15,730 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:15,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:04:15,784 [287] [DEBUG] [app] Starting request: urn:request:639f6a55-0dc0-482e-a537-db59d9632d9e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:15,784 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:15,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:15,788 [292] [DEBUG] [app] Starting request: urn:request:e3dce83a-0d28-4859-8c92-34214cd5e460 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:15,788 [292] [DEBUG] [app] Ending request: urn:request:e3dce83a-0d28-4859-8c92-34214cd5e460 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e3dce83a-0d28-4859-8c92-34214cd5e460', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:04:15,788 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:15,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:15,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:15,792 [286] [DEBUG] [app] Starting request: urn:request:23d8058e-d7e1-4c71-93a9-57ba07ece659 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:15,792 [286] [DEBUG] [app] Ending request: urn:request:23d8058e-d7e1-4c71-93a9-57ba07ece659 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:23d8058e-d7e1-4c71-93a9-57ba07ece659', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:15,792 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:04:15,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:15,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:15,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:15,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:15,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:15,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:15,810 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:15,813 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:15,816 [287] [DEBUG] [app] Ending request: urn:request:639f6a55-0dc0-482e-a537-db59d9632d9e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:639f6a55-0dc0-482e-a537-db59d9632d9e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:15,816 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:15,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:04:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:27.122849+00:00 (in 9.999625 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:27 GMT)" (scheduled at 2025-11-04 09:04:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 17, 123504), True, datetime.datetime(2025, 11, 4, 9, 4, 17, 123504), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:04:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:04:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:45.479769+00:00 (in 28.000174 seconds) securityworker stdout | 2025-11-04 09:04:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:47 GMT)" (scheduled at 2025-11-04 09:04:17.479080+00:00) securityworker stdout | 2025-11-04 09:04:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:04:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:04:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:04:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:04:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:04:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:04:17,640 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:18,296 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:04:19,609 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:04:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:04:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:51.226294+00:00 (in 29.999552 seconds) autopruneworker stdout | 2025-11-04 09:04:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:51 GMT)" (scheduled at 2025-11-04 09:04:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:04:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247001234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:04:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:04:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:04:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:04:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:04:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:32.435813+00:00 (in 11.000997 seconds) namespacegcworker stdout | 2025-11-04 09:04:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:21 GMT)" (scheduled at 2025-11-04 09:04:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:04:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:04:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 21, 435093), True, datetime.datetime(2025, 11, 4, 9, 4, 21, 435093), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:04:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:04:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:04:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:04:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:31.530064+00:00 (in 9.999615 seconds) notificationworker stdout | 2025-11-04 09:04:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:31 GMT)" (scheduled at 2025-11-04 09:04:21.530064+00:00) notificationworker stdout | 2025-11-04 09:04:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 21, 530723), True, datetime.datetime(2025, 11, 4, 9, 4, 21, 530723), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:04:21,878 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:04:23,516 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:04:23,530 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:04:23,530 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:04:23,540 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:24,743 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:04:26,967 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:37.122849+00:00 (in 9.999619 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:37 GMT)" (scheduled at 2025-11-04 09:04:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 27, 123587), True, datetime.datetime(2025, 11, 4, 9, 4, 27, 123587), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:04:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:04:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:57.222846+00:00 (in 29.999617 seconds) buildlogsarchiver stdout | 2025-11-04 09:04:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:57 GMT)" (scheduled at 2025-11-04 09:04:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:04:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 4, 27, 223438), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:04:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:04:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:04:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:04:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:04:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:04:27,694 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:04:28,341 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:04:28,732 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:29,016 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:29,191 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:29,318 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:29,346 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:04:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:04:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:44.351216+00:00 (in 14.999611 seconds) quotatotalworker stdout | 2025-11-04 09:04:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:44 GMT)" (scheduled at 2025-11-04 09:04:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:04:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:04:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:04:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:04:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:04:29,408 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:30,696 [285] [DEBUG] [app] Starting request: urn:request:b72e7449-dd92-4bf7-ab5f-688e247b52b4 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:30,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:30,701 [292] [DEBUG] [app] Starting request: urn:request:2cb983e3-f4b9-4493-baeb-a2fbd3fa6253 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:30,701 [292] [DEBUG] [app] Ending request: urn:request:2cb983e3-f4b9-4493-baeb-a2fbd3fa6253 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2cb983e3-f4b9-4493-baeb-a2fbd3fa6253', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:30,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:04:30,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:30,705 [287] [DEBUG] [app] Starting request: urn:request:27918c88-3752-4165-aab8-44fc7affaa69 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:30,705 [287] [DEBUG] [app] Ending request: urn:request:27918c88-3752-4165-aab8-44fc7affaa69 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:27918c88-3752-4165-aab8-44fc7affaa69', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:30,730 [285] [DEBUG] [app] Ending request: urn:request:b72e7449-dd92-4bf7-ab5f-688e247b52b4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b72e7449-dd92-4bf7-ab5f-688e247b52b4', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:04:30,784 [286] [DEBUG] [app] Starting request: urn:request:8f16d90a-8742-4f91-9e96-0930abf43ca3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:30,785 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:30,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:30,788 [292] [DEBUG] [app] Starting request: urn:request:7f290a35-b478-409c-9b63-d60b9975913a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:30,788 [292] [DEBUG] [app] Ending request: urn:request:7f290a35-b478-409c-9b63-d60b9975913a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7f290a35-b478-409c-9b63-d60b9975913a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:04:30,788 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:30,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:30,791 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:30,792 [287] [DEBUG] [app] Starting request: urn:request:16aa5f04-2f77-4fe2-b0c2-ba658e5ed6e2 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:30,792 [287] [DEBUG] [app] Ending request: urn:request:16aa5f04-2f77-4fe2-b0c2-ba658e5ed6e2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:16aa5f04-2f77-4fe2-b0c2-ba658e5ed6e2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:30,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:04:30,792 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:30,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:30,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:30,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:30,800 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:30,800 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:30,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:30,813 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:30,816 [286] [DEBUG] [app] Ending request: urn:request:8f16d90a-8742-4f91-9e96-0930abf43ca3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8f16d90a-8742-4f91-9e96-0930abf43ca3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:30,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:30,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:30,969 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:04:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:40.533243+00:00 (in 9.002708 seconds) notificationworker stdout | 2025-11-04 09:04:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:41 GMT)" (scheduled at 2025-11-04 09:04:31.530064+00:00) notificationworker stdout | 2025-11-04 09:04:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 31, 530808), True, datetime.datetime(2025, 11, 4, 9, 4, 31, 530808), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:04:31,770 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:04:31,959 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:04:31,970 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:04:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:04:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:21.434460+00:00 (in 48.998228 seconds) namespacegcworker stdout | 2025-11-04 09:04:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:32 GMT)" (scheduled at 2025-11-04 09:04:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:04:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:04:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:04:33,261 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:04:33,267 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:04:33,363 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:04:33,540 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:04:33,554 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:04:33,554 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:04:33,564 [66] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:04:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:04:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:48.625192+00:00 (in 15.001844 seconds) exportactionlogsworker stdout | 2025-11-04 09:04:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:33 GMT)" (scheduled at 2025-11-04 09:04:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:04:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:04:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 33, 623591), True, datetime.datetime(2025, 11, 4, 9, 4, 33, 623591), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:04:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:04:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:04:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:33 GMT)" executed successfully builder stdout | 2025-11-04 09:04:34,150 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:04:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:04:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:38.437104+00:00 (in 3.000913 seconds) gcworker stdout | 2025-11-04 09:04:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:05 GMT)" (scheduled at 2025-11-04 09:04:35.435809+00:00) gcworker stdout | 2025-11-04 09:04:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037475436, 1209600, True, 3, 500, 1, 0]) queuecleanupworker stdout | 2025-11-04 09:04:35,441 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:04:35,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:04:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:04:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:05 GMT)" executed successfully servicekey stdout | 2025-11-04 09:04:35,805 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:04:36,435 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:04:36,820 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:40.124445+00:00 (in 3.001142 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:47 GMT)" (scheduled at 2025-11-04 09:04:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 37, 123599), True, datetime.datetime(2025, 11, 4, 9, 4, 37, 123599), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:04:37,647 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,655 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,692 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,706 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,707 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,707 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,749 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,876 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:04:37,898 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:04:38,153 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:04:38,153 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:04:38,164 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:04:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:04:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:05.435809+00:00 (in 26.998311 seconds) gcworker stdout | 2025-11-04 09:04:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:08 GMT)" (scheduled at 2025-11-04 09:04:38.437104+00:00) gcworker stdout | 2025-11-04 09:04:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:04:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247018448, None, 1, 0]) gcworker stdout | 2025-11-04 09:04:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:04:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:04:39,003 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:04:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:04:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:01.922827+00:00 (in 21.997782 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:04:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:39 GMT)" (scheduled at 2025-11-04 09:04:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:04:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:04:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:04:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:47.122849+00:00 (in 6.998013 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:40 GMT)" (scheduled at 2025-11-04 09:04:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:04:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:04:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:41.530064+00:00 (in 0.996333 seconds) notificationworker stdout | 2025-11-04 09:04:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:40 GMT)" (scheduled at 2025-11-04 09:04:40.533243+00:00) notificationworker stdout | 2025-11-04 09:04:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 4, 40, 534007), 'notification/%']) notificationworker stdout | 2025-11-04 09:04:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 40, 534007), True, datetime.datetime(2025, 11, 4, 9, 4, 40, 534007), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:04:40,549 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 40, 534007), True, datetime.datetime(2025, 11, 4, 9, 4, 40, 534007), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 4, 40, 534007), 'notification/%']) notificationworker stdout | 2025-11-04 09:04:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:04:40,765 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:04:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:51.530064+00:00 (in 9.999550 seconds) notificationworker stdout | 2025-11-04 09:04:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:51 GMT)" (scheduled at 2025-11-04 09:04:41.530064+00:00) notificationworker stdout | 2025-11-04 09:04:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 41, 530787), True, datetime.datetime(2025, 11, 4, 9, 4, 41, 530787), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:04:42,209 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:04:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:57.523915+00:00 (in 14.090643 seconds) repositorygcworker stdout | 2025-11-04 09:04:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:43 GMT)" (scheduled at 2025-11-04 09:04:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:04:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:04:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 43, 433587), True, datetime.datetime(2025, 11, 4, 9, 4, 43, 433587), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:04:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:04:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:04:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:43 GMT)" executed successfully builder stdout | 2025-11-04 09:04:43,564 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:04:43,577 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:04:43,577 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:04:43,588 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:04:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:04:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:59.351216+00:00 (in 14.999581 seconds) quotatotalworker stdout | 2025-11-04 09:04:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:59 GMT)" (scheduled at 2025-11-04 09:04:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:04:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:04:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:04:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:04:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:04:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:04:44,716 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:04:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:04:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:47.479080+00:00 (in 1.998871 seconds) securityworker stdout | 2025-11-04 09:04:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:15 GMT)" (scheduled at 2025-11-04 09:04:45.479769+00:00) securityworker stdout | 2025-11-04 09:04:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:04:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:04:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:04:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 45, 483630), 1, 2]) securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 45, 483630), 1, 2]) securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:04:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:04:45,696 [288] [DEBUG] [app] Starting request: urn:request:c4159887-7e2f-4a8d-a450-9abd9ba5caa8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:45,701 [292] [DEBUG] [app] Starting request: urn:request:226ed1f5-fcb4-4771-99f4-9ee0476668f5 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:45,701 [292] [DEBUG] [app] Ending request: urn:request:226ed1f5-fcb4-4771-99f4-9ee0476668f5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:226ed1f5-fcb4-4771-99f4-9ee0476668f5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:04:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:45,706 [285] [DEBUG] [app] Starting request: urn:request:198a8a5a-3eff-4f85-8afa-f1dc3195c1c3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:45,706 [285] [DEBUG] [app] Ending request: urn:request:198a8a5a-3eff-4f85-8afa-f1dc3195c1c3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:198a8a5a-3eff-4f85-8afa-f1dc3195c1c3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:45,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:45,728 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:45,731 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:45,734 [288] [DEBUG] [app] Ending request: urn:request:c4159887-7e2f-4a8d-a450-9abd9ba5caa8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c4159887-7e2f-4a8d-a450-9abd9ba5caa8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:45,734 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:45,735 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.039 47 0.039) gunicorn-web stdout | 2025-11-04 09:04:45,784 [285] [DEBUG] [app] Starting request: urn:request:e44e9a99-4f94-4330-883b-4f06d40da95a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:04:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:04:45,788 [292] [DEBUG] [app] Starting request: urn:request:760a169d-7b68-44c1-abe7-af39c0e7e3a3 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:04:45,788 [292] [DEBUG] [app] Ending request: urn:request:760a169d-7b68-44c1-abe7-af39c0e7e3a3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:760a169d-7b68-44c1-abe7-af39c0e7e3a3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:04:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:04:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:04:45,792 [287] [DEBUG] [app] Starting request: urn:request:2314b2e2-f89b-4912-bb3c-6a53c7283cfc (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:04:45,792 [287] [DEBUG] [app] Ending request: urn:request:2314b2e2-f89b-4912-bb3c-6a53c7283cfc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2314b2e2-f89b-4912-bb3c-6a53c7283cfc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:45,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:04:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:04:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:04:45,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:45,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:45,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:45,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:45,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:45,816 [285] [DEBUG] [app] Ending request: urn:request:e44e9a99-4f94-4330-883b-4f06d40da95a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e44e9a99-4f94-4330-883b-4f06d40da95a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:45,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:45,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:04:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:04:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:04:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:53.124163+00:00 (in 6.000885 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:57 GMT)" (scheduled at 2025-11-04 09:04:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 47, 123626), True, datetime.datetime(2025, 11, 4, 9, 4, 47, 123626), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:04:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:04:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:15.479769+00:00 (in 28.000295 seconds) securityworker stdout | 2025-11-04 09:04:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:17 GMT)" (scheduled at 2025-11-04 09:04:47.479080+00:00) securityworker stdout | 2025-11-04 09:04:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:04:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:04:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:04:47,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:04:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:04:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:47,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 47, 484129), 1, 2]) securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 47, 484129), 1, 2]) securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:04:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:17 GMT)" executed successfully securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:04:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityscanningnotificationworker stdout | 2025-11-04 09:04:47,648 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:48,305 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:04:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:04:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:33.622985+00:00 (in 44.997366 seconds) exportactionlogsworker stdout | 2025-11-04 09:04:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:48 GMT)" (scheduled at 2025-11-04 09:04:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:04:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:04:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:04:49,618 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:04:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:04:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:21.226294+00:00 (in 29.999584 seconds) autopruneworker stdout | 2025-11-04 09:04:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:21 GMT)" (scheduled at 2025-11-04 09:04:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:04:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247031234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:04:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:04:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:04:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:04:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:01.530064+00:00 (in 9.999552 seconds) notificationworker stdout | 2025-11-04 09:04:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:01 GMT)" (scheduled at 2025-11-04 09:04:51.530064+00:00) notificationworker stdout | 2025-11-04 09:04:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 51, 530767), True, datetime.datetime(2025, 11, 4, 9, 4, 51, 530767), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:04:51,886 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:57.122849+00:00 (in 3.998232 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:53 GMT)" (scheduled at 2025-11-04 09:04:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 4, 53, 124901), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:04:53,137 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 53, 124901), True, datetime.datetime(2025, 11, 4, 9, 4, 53, 124901), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:04:53,141 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 53, 124901), True, datetime.datetime(2025, 11, 4, 9, 4, 53, 124901), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 4, 53, 124901), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:04:53,144 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:53,144 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:53 GMT)" executed successfully builder stdout | 2025-11-04 09:04:53,588 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:04:53,603 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:04:53,603 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:04:53,615 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:54,752 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:04:56,976 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:07.122849+00:00 (in 9.999619 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:07 GMT)" (scheduled at 2025-11-04 09:04:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 57, 123569), True, datetime.datetime(2025, 11, 4, 9, 4, 57, 123569), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:04:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:09.133645+00:00 (in 12.001469 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:57 GMT)" (scheduled at 2025-11-04 09:04:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:04:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:04:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:04:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:04:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:04:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:27.222846+00:00 (in 29.999364 seconds) buildlogsarchiver stdout | 2025-11-04 09:04:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:27 GMT)" (scheduled at 2025-11-04 09:04:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:04:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 4, 57, 223764), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:04:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:04:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:04:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:04:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:04:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:04:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:43.432867+00:00 (in 45.908578 seconds) repositorygcworker stdout | 2025-11-04 09:04:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:57 GMT)" (scheduled at 2025-11-04 09:04:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:04:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:04:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:04:57,703 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:04:58,350 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:04:58,741 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:59,026 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:59,203 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:04:59,332 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:04:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:04:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:14.351216+00:00 (in 14.999561 seconds) quotatotalworker stdout | 2025-11-04 09:04:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:14 GMT)" (scheduled at 2025-11-04 09:04:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:04:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:04:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) gunicorn-web stdout | 2025-11-04 09:04:59,360 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:04:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:04:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:04:59,423 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:00,697 [285] [DEBUG] [app] Starting request: urn:request:61c1b950-0fe7-48aa-b912-df464aeecfaa (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:00,701 [292] [DEBUG] [app] Starting request: urn:request:80a0acba-0756-45be-9243-ce74e440f339 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:00,702 [292] [DEBUG] [app] Ending request: urn:request:80a0acba-0756-45be-9243-ce74e440f339 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:80a0acba-0756-45be-9243-ce74e440f339', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:05:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:00,706 [288] [DEBUG] [app] Starting request: urn:request:8bbd8789-18df-4100-a1fd-f816a12b1697 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:00,706 [288] [DEBUG] [app] Ending request: urn:request:8bbd8789-18df-4100-a1fd-f816a12b1697 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8bbd8789-18df-4100-a1fd-f816a12b1697', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:05:00,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:00,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:00,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:00,726 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:00,729 [285] [DEBUG] [app] Ending request: urn:request:61c1b950-0fe7-48aa-b912-df464aeecfaa (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:61c1b950-0fe7-48aa-b912-df464aeecfaa', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:00,729 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:00,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) gunicorn-web stdout | 2025-11-04 09:05:00,783 [285] [DEBUG] [app] Starting request: urn:request:4de1330f-3b27-4287-ba5f-4132e42b5afd (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:00,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:00,785 [298] [DEBUG] [app] Starting request: urn:request:d5079230-ed8b-4372-ad28-1ea5bdb1177d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:00,785 [298] [DEBUG] [app] Ending request: urn:request:d5079230-ed8b-4372-ad28-1ea5bdb1177d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d5079230-ed8b-4372-ad28-1ea5bdb1177d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:00,785 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:00,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:00,789 [288] [DEBUG] [app] Starting request: urn:request:aee3564a-6d14-4760-8a08-1adda33f3f50 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:00,790 [288] [DEBUG] [app] Ending request: urn:request:aee3564a-6d14-4760-8a08-1adda33f3f50 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:aee3564a-6d14-4760-8a08-1adda33f3f50', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:00,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:00,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:00,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:00,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:00,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:00,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:00,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:00,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:00,813 [285] [DEBUG] [app] Ending request: urn:request:4de1330f-3b27-4287-ba5f-4132e42b5afd (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4de1330f-3b27-4287-ba5f-4132e42b5afd', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:00,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:00,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,978 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:05:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:11.530064+00:00 (in 9.999589 seconds) notificationworker stdout | 2025-11-04 09:05:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:11 GMT)" (scheduled at 2025-11-04 09:05:01.530064+00:00) notificationworker stdout | 2025-11-04 09:05:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 1, 530779), True, datetime.datetime(2025, 11, 4, 9, 5, 1, 530779), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:05:01,779 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:05:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:05:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:39.924652+00:00 (in 38.001444 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:05:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:01 GMT)" (scheduled at 2025-11-04 09:05:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:05:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:05:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 1, 923456), True, datetime.datetime(2025, 11, 4, 9, 5, 1, 923456), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:05:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:05:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:05:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:01 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:05:01,967 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:05:01,979 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:05:03,271 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:05:03,276 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:05:03,372 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:05:03,615 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:05:03,629 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:05:03,629 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:05:03,639 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:05:04,159 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:08.437104+00:00 (in 3.000931 seconds) gcworker stdout | 2025-11-04 09:05:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:35 GMT)" (scheduled at 2025-11-04 09:05:05.435809+00:00) gcworker stdout | 2025-11-04 09:05:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037505436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:05:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:05:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:35 GMT)" executed successfully queuecleanupworker stdout | 2025-11-04 09:05:05,450 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:05:05,814 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:06,444 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:05:06,829 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:17.122849+00:00 (in 9.999610 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:17 GMT)" (scheduled at 2025-11-04 09:05:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 7, 123536), True, datetime.datetime(2025, 11, 4, 9, 5, 7, 123536), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:05:07,657 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,665 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,704 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,717 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,721 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,721 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,760 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,889 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:07,911 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:08,164 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:08,164 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:08,174 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:35.435809+00:00 (in 26.998303 seconds) gcworker stdout | 2025-11-04 09:05:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:38 GMT)" (scheduled at 2025-11-04 09:05:08.437104+00:00) gcworker stdout | 2025-11-04 09:05:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:05:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247048449, None, 1, 0]) gcworker stdout | 2025-11-04 09:05:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:05:09,012 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:05:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:05:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:57.131795+00:00 (in 47.997738 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:09 GMT)" (scheduled at 2025-11-04 09:05:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:05:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:05:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:05:10,774 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:05:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:15.535095+00:00 (in 4.004617 seconds) notificationworker stdout | 2025-11-04 09:05:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:21 GMT)" (scheduled at 2025-11-04 09:05:11.530064+00:00) notificationworker stdout | 2025-11-04 09:05:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 11, 530759), True, datetime.datetime(2025, 11, 4, 9, 5, 11, 530759), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:21 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:05:12,218 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:05:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:05:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:12.222828+00:00 (in 59.999592 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:05:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:12 GMT)" (scheduled at 2025-11-04 09:05:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:05:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:05:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:05:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:12 GMT)" executed successfully builder stdout | 2025-11-04 09:05:13,639 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:05:13,652 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:05:13,652 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:05:13,662 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:05:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:05:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:29.351216+00:00 (in 14.999595 seconds) quotatotalworker stdout | 2025-11-04 09:05:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:29 GMT)" (scheduled at 2025-11-04 09:05:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:05:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:05:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:05:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:05:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:05:14,725 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:17.479080+00:00 (in 1.998841 seconds) securityworker stdout | 2025-11-04 09:05:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:45 GMT)" (scheduled at 2025-11-04 09:05:15.479769+00:00) securityworker stdout | 2025-11-04 09:05:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:05:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:05:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 15, 483992), 1, 2]) securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:05:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 15, 483992), 1, 2]) securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:05:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:45 GMT)" executed successfully securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker notificationworker stdout | 2025-11-04 09:05:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:21.530064+00:00 (in 5.994524 seconds) notificationworker stdout | 2025-11-04 09:05:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:15 GMT)" (scheduled at 2025-11-04 09:05:15.535095+00:00) notificationworker stdout | 2025-11-04 09:05:15,536 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:05:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:05:15,696 [288] [DEBUG] [app] Starting request: urn:request:9d0951c6-b5bb-4ae1-a158-2f74f91a0de5 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:15,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:15,701 [299] [DEBUG] [app] Starting request: urn:request:fb928178-0d7f-420a-a38c-afa45eeaef96 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:15,701 [299] [DEBUG] [app] Ending request: urn:request:fb928178-0d7f-420a-a38c-afa45eeaef96 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fb928178-0d7f-420a-a38c-afa45eeaef96', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:15,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:05:15,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:15,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:15,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:15,705 [285] [DEBUG] [app] Starting request: urn:request:eb2187c2-e3e9-4058-bc3c-822959b47de6 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:15,705 [285] [DEBUG] [app] Ending request: urn:request:eb2187c2-e3e9-4058-bc3c-822959b47de6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:eb2187c2-e3e9-4058-bc3c-822959b47de6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:15,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:15,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:15,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:15,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:15,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:15,726 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:15,729 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:15,732 [288] [DEBUG] [app] Ending request: urn:request:9d0951c6-b5bb-4ae1-a158-2f74f91a0de5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9d0951c6-b5bb-4ae1-a158-2f74f91a0de5', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:15,732 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:15,733 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.037 47 0.037) gunicorn-web stdout | 2025-11-04 09:05:15,784 [285] [DEBUG] [app] Starting request: urn:request:bac51b94-0e7d-45f5-9733-870d2061f3b7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:15,788 [292] [DEBUG] [app] Starting request: urn:request:da8ff18f-3a19-4f58-9884-2145b7e790cb (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:15,788 [292] [DEBUG] [app] Ending request: urn:request:da8ff18f-3a19-4f58-9884-2145b7e790cb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:da8ff18f-3a19-4f58-9884-2145b7e790cb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:15,792 [287] [DEBUG] [app] Starting request: urn:request:0e0d82a4-5ef3-4058-b93a-71f6f360c3d5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:15,792 [287] [DEBUG] [app] Ending request: urn:request:0e0d82a4-5ef3-4058-b93a-71f6f360c3d5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0e0d82a4-5ef3-4058-b93a-71f6f360c3d5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:15,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:15,816 [285] [DEBUG] [app] Ending request: urn:request:bac51b94-0e7d-45f5-9733-870d2061f3b7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bac51b94-0e7d-45f5-9733-870d2061f3b7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:15,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:05:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:05:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:27.122849+00:00 (in 9.999573 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:27 GMT)" (scheduled at 2025-11-04 09:05:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 17, 123548), True, datetime.datetime(2025, 11, 4, 9, 5, 17, 123548), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:05:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:45.479769+00:00 (in 28.000214 seconds) securityworker stdout | 2025-11-04 09:05:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:47 GMT)" (scheduled at 2025-11-04 09:05:17.479080+00:00) securityworker stdout | 2025-11-04 09:05:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:05:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:05:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:05:17,657 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:18,314 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:05:19,627 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:05:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:05:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:51.226294+00:00 (in 29.999548 seconds) autopruneworker stdout | 2025-11-04 09:05:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:51 GMT)" (scheduled at 2025-11-04 09:05:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:05:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247061234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:05:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:05:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:05:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:05:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:05:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:32.435813+00:00 (in 11.000962 seconds) namespacegcworker stdout | 2025-11-04 09:05:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:21 GMT)" (scheduled at 2025-11-04 09:05:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:05:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:05:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 21, 435096), True, datetime.datetime(2025, 11, 4, 9, 5, 21, 435096), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:05:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:05:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:05:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:05:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:31.530064+00:00 (in 9.999646 seconds) notificationworker stdout | 2025-11-04 09:05:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:31 GMT)" (scheduled at 2025-11-04 09:05:21.530064+00:00) notificationworker stdout | 2025-11-04 09:05:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 21, 530671), True, datetime.datetime(2025, 11, 4, 9, 5, 21, 530671), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:05:21,895 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:05:23,662 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:05:23,676 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:05:23,676 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:05:23,686 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:24,761 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:05:26,985 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:37.122849+00:00 (in 9.999577 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:37 GMT)" (scheduled at 2025-11-04 09:05:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 27, 123581), True, datetime.datetime(2025, 11, 4, 9, 5, 27, 123581), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:05:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:05:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:57.222846+00:00 (in 29.999575 seconds) buildlogsarchiver stdout | 2025-11-04 09:05:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:57 GMT)" (scheduled at 2025-11-04 09:05:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:05:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 5, 27, 223505), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:05:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:05:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:05:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:05:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:05:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:05:27,711 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:05:28,359 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:05:28,750 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:29,036 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:29,215 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:29,345 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:05:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:05:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:44.351216+00:00 (in 14.999571 seconds) quotatotalworker stdout | 2025-11-04 09:05:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:44 GMT)" (scheduled at 2025-11-04 09:05:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:05:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:05:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:05:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:05:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:05:29,373 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:29,437 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:30,697 [288] [DEBUG] [app] Starting request: urn:request:9fc8c149-ee62-4d90-b1c4-e01e470cb954 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:30,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:30,701 [299] [DEBUG] [app] Starting request: urn:request:83a36841-e8f5-47e1-ac7f-1a2e7781ea44 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:30,702 [299] [DEBUG] [app] Ending request: urn:request:83a36841-e8f5-47e1-ac7f-1a2e7781ea44 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:83a36841-e8f5-47e1-ac7f-1a2e7781ea44', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:30,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:30,706 [287] [DEBUG] [app] Starting request: urn:request:c85912de-9810-41f1-b41c-c86d35728a70 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:30,706 [287] [DEBUG] [app] Ending request: urn:request:c85912de-9810-41f1-b41c-c86d35728a70 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c85912de-9810-41f1-b41c-c86d35728a70', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:30,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:30,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:30,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:30,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:30,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:30,731 [288] [DEBUG] [app] Ending request: urn:request:9fc8c149-ee62-4d90-b1c4-e01e470cb954 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9fc8c149-ee62-4d90-b1c4-e01e470cb954', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:30,731 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:05:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:05:30,784 [287] [DEBUG] [app] Starting request: urn:request:f8f65cb6-f291-4605-b027-67139e6cfbc9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:30,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:30,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:30,788 [299] [DEBUG] [app] Starting request: urn:request:97e92faa-beed-4831-9436-48859c1f5942 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:30,788 [299] [DEBUG] [app] Ending request: urn:request:97e92faa-beed-4831-9436-48859c1f5942 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:97e92faa-beed-4831-9436-48859c1f5942', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:30,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:30,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:30,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:30,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:30,792 [288] [DEBUG] [app] Starting request: urn:request:73745655-3f9f-4dfd-acba-49e9524148ff (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:30,792 [288] [DEBUG] [app] Ending request: urn:request:73745655-3f9f-4dfd-acba-49e9524148ff (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:73745655-3f9f-4dfd-acba-49e9524148ff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:30,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:30,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:30,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:30,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:30,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:30,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:30,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:30,811 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:30,814 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:30,817 [287] [DEBUG] [app] Ending request: urn:request:f8f65cb6-f291-4605-b027-67139e6cfbc9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f8f65cb6-f291-4605-b027-67139e6cfbc9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:30,817 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:30,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:30,987 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:05:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:41.530064+00:00 (in 9.999559 seconds) notificationworker stdout | 2025-11-04 09:05:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:41 GMT)" (scheduled at 2025-11-04 09:05:31.530064+00:00) notificationworker stdout | 2025-11-04 09:05:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 31, 530813), True, datetime.datetime(2025, 11, 4, 9, 5, 31, 530813), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:05:31,788 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:05:31,987 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:05:31,988 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:05:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:05:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:21.434460+00:00 (in 48.998226 seconds) namespacegcworker stdout | 2025-11-04 09:05:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:32 GMT)" (scheduled at 2025-11-04 09:05:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:05:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:05:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:05:33,280 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:05:33,284 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:05:33,381 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:05:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:05:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:48.625192+00:00 (in 15.001768 seconds) exportactionlogsworker stdout | 2025-11-04 09:05:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:33 GMT)" (scheduled at 2025-11-04 09:05:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:05:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:05:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 33, 623763), True, datetime.datetime(2025, 11, 4, 9, 5, 33, 623763), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:05:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:05:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:05:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:33 GMT)" executed successfully builder stdout | 2025-11-04 09:05:33,686 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:05:33,700 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:05:33,700 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:05:33,710 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:05:34,169 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:38.437104+00:00 (in 3.000939 seconds) gcworker stdout | 2025-11-04 09:05:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:05 GMT)" (scheduled at 2025-11-04 09:05:35.435809+00:00) gcworker stdout | 2025-11-04 09:05:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037535436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:05:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:05:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:05 GMT)" executed successfully queuecleanupworker stdout | 2025-11-04 09:05:35,459 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:05:35,823 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:36,454 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:05:36,838 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:40.124445+00:00 (in 3.001190 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:47 GMT)" (scheduled at 2025-11-04 09:05:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 37, 123551), True, datetime.datetime(2025, 11, 4, 9, 5, 37, 123551), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:05:37,671 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,675 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,715 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,731 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,734 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,734 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,771 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,901 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:37,923 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:38,175 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:38,175 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:38,184 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:05.435809+00:00 (in 26.998340 seconds) gcworker stdout | 2025-11-04 09:05:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:08 GMT)" (scheduled at 2025-11-04 09:05:38.437104+00:00) gcworker stdout | 2025-11-04 09:05:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:05:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247078449, None, 1, 0]) gcworker stdout | 2025-11-04 09:05:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:05:39,020 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:05:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:05:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:01.922827+00:00 (in 21.997731 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:05:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:39 GMT)" (scheduled at 2025-11-04 09:05:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:05:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:05:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:05:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:47.122849+00:00 (in 6.998016 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:40 GMT)" (scheduled at 2025-11-04 09:05:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:05:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:05:40,783 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:05:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:51.530064+00:00 (in 9.999597 seconds) notificationworker stdout | 2025-11-04 09:05:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:51 GMT)" (scheduled at 2025-11-04 09:05:41.530064+00:00) notificationworker stdout | 2025-11-04 09:05:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 41, 530771), True, datetime.datetime(2025, 11, 4, 9, 5, 41, 530771), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:05:42,227 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:05:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:05:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:57.523915+00:00 (in 14.090658 seconds) repositorygcworker stdout | 2025-11-04 09:05:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:43 GMT)" (scheduled at 2025-11-04 09:05:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:05:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:05:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 43, 433521), True, datetime.datetime(2025, 11, 4, 9, 5, 43, 433521), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:05:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:05:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:05:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:43 GMT)" executed successfully builder stdout | 2025-11-04 09:05:43,710 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:05:43,723 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:05:43,723 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:05:43,733 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:05:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:05:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:59.351216+00:00 (in 14.999568 seconds) quotatotalworker stdout | 2025-11-04 09:05:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:59 GMT)" (scheduled at 2025-11-04 09:05:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:05:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:05:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:05:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:05:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:05:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:05:44,735 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:47.479080+00:00 (in 1.998840 seconds) securityworker stdout | 2025-11-04 09:05:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:15 GMT)" (scheduled at 2025-11-04 09:05:45.479769+00:00) securityworker stdout | 2025-11-04 09:05:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:05:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 45, 483551), 1, 2]) securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:45,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 45, 483551), 1, 2]) securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:05:45,696 [286] [DEBUG] [app] Starting request: urn:request:8271a49b-d280-4f1f-af02-75e17c3f669c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:45,697 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:45,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:45,701 [298] [DEBUG] [app] Starting request: urn:request:70300916-f8c4-42ff-be7d-5e235e942315 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:45,701 [298] [DEBUG] [app] Ending request: urn:request:70300916-f8c4-42ff-be7d-5e235e942315 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:70300916-f8c4-42ff-be7d-5e235e942315', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:05:45,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:45,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:45,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:45,704 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:45,705 [285] [DEBUG] [app] Starting request: urn:request:500e9b39-1e8f-49ff-a219-43ec1cab89f4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:45,705 [285] [DEBUG] [app] Ending request: urn:request:500e9b39-1e8f-49ff-a219-43ec1cab89f4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:500e9b39-1e8f-49ff-a219-43ec1cab89f4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:45,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:45,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:45,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:45,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:45,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:45,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:45,723 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:45,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:45,730 [286] [DEBUG] [app] Ending request: urn:request:8271a49b-d280-4f1f-af02-75e17c3f669c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8271a49b-d280-4f1f-af02-75e17c3f669c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:45,730 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:05:45,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:05:45,783 [285] [DEBUG] [app] Starting request: urn:request:5963ade5-5992-4d1f-ae1b-98e75720bd10 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:05:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:45,786 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:05:45,787 [299] [DEBUG] [app] Starting request: urn:request:cff2a5e5-6a53-47dd-8329-ce327cdcc68a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:05:45,787 [299] [DEBUG] [app] Ending request: urn:request:cff2a5e5-6a53-47dd-8329-ce327cdcc68a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cff2a5e5-6a53-47dd-8329-ce327cdcc68a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:45,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:05:45,792 [288] [DEBUG] [app] Starting request: urn:request:84575d06-97c8-4f39-b9e7-14c6cf90e66d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:05:45,792 [288] [DEBUG] [app] Ending request: urn:request:84575d06-97c8-4f39-b9e7-14c6cf90e66d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:84575d06-97c8-4f39-b9e7-14c6cf90e66d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:05:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:05:45,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:05:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:45,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:45,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:45,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:45,809 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:45,812 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:45,815 [285] [DEBUG] [app] Ending request: urn:request:5963ade5-5992-4d1f-ae1b-98e75720bd10 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5963ade5-5992-4d1f-ae1b-98e75720bd10', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:45,815 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:45,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:05:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:05:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.033) proxycacheblobworker stdout | 2025-11-04 09:05:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:57.122849+00:00 (in 9.999633 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:57 GMT)" (scheduled at 2025-11-04 09:05:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 47, 123476), True, datetime.datetime(2025, 11, 4, 9, 5, 47, 123476), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:05:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:15.479769+00:00 (in 28.000224 seconds) securityworker stdout | 2025-11-04 09:05:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:17 GMT)" (scheduled at 2025-11-04 09:05:47.479080+00:00) securityworker stdout | 2025-11-04 09:05:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:05:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:05:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:05:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 47, 483000), 1, 2]) securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 47, 483000), 1, 2]) securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:05:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:05:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:05:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:05:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:05:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:05:47,666 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:48,323 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:05:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:05:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:33.622985+00:00 (in 44.997376 seconds) exportactionlogsworker stdout | 2025-11-04 09:05:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:48 GMT)" (scheduled at 2025-11-04 09:05:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:05:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:05:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:05:49,635 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:05:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:05:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:21.226294+00:00 (in 29.999538 seconds) autopruneworker stdout | 2025-11-04 09:05:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:21 GMT)" (scheduled at 2025-11-04 09:05:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:05:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247091235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:05:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:05:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:05:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:05:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:01.530064+00:00 (in 9.999540 seconds) notificationworker stdout | 2025-11-04 09:05:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:01 GMT)" (scheduled at 2025-11-04 09:05:51.530064+00:00) notificationworker stdout | 2025-11-04 09:05:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 51, 530802), True, datetime.datetime(2025, 11, 4, 9, 5, 51, 530802), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:05:51,915 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:05:53,733 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:05:53,746 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:05:53,746 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:05:53,756 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:54,770 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:05:56,994 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:07.122849+00:00 (in 9.999636 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:07 GMT)" (scheduled at 2025-11-04 09:05:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 57, 123457), True, datetime.datetime(2025, 11, 4, 9, 5, 57, 123457), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:05:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:09.133645+00:00 (in 12.001476 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:57 GMT)" (scheduled at 2025-11-04 09:05:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:05:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:05:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:05:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:05:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:05:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:27.222846+00:00 (in 29.999627 seconds) buildlogsarchiver stdout | 2025-11-04 09:05:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:27 GMT)" (scheduled at 2025-11-04 09:05:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:05:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 5, 57, 223418), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:05:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:05:57,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:05:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:05:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:05:57,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:05:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:05:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:43.432867+00:00 (in 45.908576 seconds) repositorygcworker stdout | 2025-11-04 09:05:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:57 GMT)" (scheduled at 2025-11-04 09:05:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:05:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:05:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:05:57,720 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:05:58,368 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:05:58,759 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:59,047 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:59,232 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:05:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:05:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:14.351216+00:00 (in 14.999581 seconds) quotatotalworker stdout | 2025-11-04 09:05:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:14 GMT)" (scheduled at 2025-11-04 09:05:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:05:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:05:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) gunicorn-web stdout | 2025-11-04 09:05:59,357 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:05:59,367 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:05:59,367 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:05:59,386 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:59,451 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:00,697 [287] [DEBUG] [app] Starting request: urn:request:3d944da7-8fcf-4d17-9935-693f619a4f5d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:00,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:00,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:00,701 [292] [DEBUG] [app] Starting request: urn:request:d12be751-475d-4107-b2f1-6df7bf9c767a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:00,701 [292] [DEBUG] [app] Ending request: urn:request:d12be751-475d-4107-b2f1-6df7bf9c767a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d12be751-475d-4107-b2f1-6df7bf9c767a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:00,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:00,701 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:00,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:00,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:00,705 [285] [DEBUG] [app] Starting request: urn:request:11048611-6434-483c-b262-10ce01a9c83c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:00,705 [285] [DEBUG] [app] Ending request: urn:request:11048611-6434-483c-b262-10ce01a9c83c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:11048611-6434-483c-b262-10ce01a9c83c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:00,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:00,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:00,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:00,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:00,713 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:00,713 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:00,723 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:00,726 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:00,729 [287] [DEBUG] [app] Ending request: urn:request:3d944da7-8fcf-4d17-9935-693f619a4f5d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3d944da7-8fcf-4d17-9935-693f619a4f5d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:00,729 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:00,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:06:00,784 [286] [DEBUG] [app] Starting request: urn:request:ae9881af-6548-46ee-84a2-41ec4aac756c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:00,785 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:00,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:00,788 [299] [DEBUG] [app] Starting request: urn:request:0b5a6bb7-8f2e-4f1e-823d-990095f846cc (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:00,789 [299] [DEBUG] [app] Ending request: urn:request:0b5a6bb7-8f2e-4f1e-823d-990095f846cc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0b5a6bb7-8f2e-4f1e-823d-990095f846cc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:00,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:00,789 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:00,790 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:00,792 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:00,793 [288] [DEBUG] [app] Starting request: urn:request:a59c8b6e-c825-475c-81da-8053dbfbe2c2 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:00,793 [288] [DEBUG] [app] Ending request: urn:request:a59c8b6e-c825-475c-81da-8053dbfbe2c2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a59c8b6e-c825-475c-81da-8053dbfbe2c2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:00,794 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:00,794 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:00,794 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:00,794 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:00,794 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:00,802 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:00,802 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:00,811 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:00,814 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:00,817 [286] [DEBUG] [app] Ending request: urn:request:ae9881af-6548-46ee-84a2-41ec4aac756c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ae9881af-6548-46ee-84a2-41ec4aac756c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:00,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:00,818 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,996 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:06:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:11.530064+00:00 (in 9.999539 seconds) notificationworker stdout | 2025-11-04 09:06:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:11 GMT)" (scheduled at 2025-11-04 09:06:01.530064+00:00) notificationworker stdout | 2025-11-04 09:06:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 1, 530784), True, datetime.datetime(2025, 11, 4, 9, 6, 1, 530784), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:06:01,797 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:06:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:06:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:39.924652+00:00 (in 38.001400 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:06:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:01 GMT)" (scheduled at 2025-11-04 09:06:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:06:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:06:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 1, 923556), True, datetime.datetime(2025, 11, 4, 9, 6, 1, 923556), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:06:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:06:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:06:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:01 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:06:01,996 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:06:01,997 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:06:03,289 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:06:03,293 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:03,390 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:06:03,757 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:06:03,770 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:06:03,770 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:06:03,780 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:06:04,178 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:06:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:08.437104+00:00 (in 3.000907 seconds) gcworker stdout | 2025-11-04 09:06:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:35 GMT)" (scheduled at 2025-11-04 09:06:05.435809+00:00) gcworker stdout | 2025-11-04 09:06:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037565436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:06:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:06:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:35 GMT)" executed successfully queuecleanupworker stdout | 2025-11-04 09:06:05,468 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:06:05,832 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:06,464 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:06:06,857 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:17.122849+00:00 (in 9.999561 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:17 GMT)" (scheduled at 2025-11-04 09:06:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 7, 123582), True, datetime.datetime(2025, 11, 4, 9, 6, 7, 123582), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:06:07,686 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,686 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,726 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,746 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,746 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,751 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,794 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,912 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:07,936 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:08,185 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:08,185 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:08,194 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:06:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:35.435809+00:00 (in 26.998330 seconds) gcworker stdout | 2025-11-04 09:06:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:38 GMT)" (scheduled at 2025-11-04 09:06:08.437104+00:00) gcworker stdout | 2025-11-04 09:06:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:06:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247108449, None, 1, 0]) gcworker stdout | 2025-11-04 09:06:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:06:09,029 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:06:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:06:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:57.131795+00:00 (in 47.997705 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:09 GMT)" (scheduled at 2025-11-04 09:06:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:06:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:06:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:06:10,799 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:06:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:15.535095+00:00 (in 4.004614 seconds) notificationworker stdout | 2025-11-04 09:06:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:21 GMT)" (scheduled at 2025-11-04 09:06:11.530064+00:00) notificationworker stdout | 2025-11-04 09:06:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 11, 530767), True, datetime.datetime(2025, 11, 4, 9, 6, 11, 530767), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:06:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:06:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:12.222828+00:00 (in 59.999516 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:06:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:12 GMT)" (scheduled at 2025-11-04 09:06:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:06:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:06:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:06:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:06:12,242 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:06:13,780 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:06:13,794 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:06:13,794 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:06:13,804 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:06:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:06:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:29.351216+00:00 (in 14.999554 seconds) quotatotalworker stdout | 2025-11-04 09:06:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:29 GMT)" (scheduled at 2025-11-04 09:06:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:06:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:06:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:06:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:06:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:06:14,744 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:17.479080+00:00 (in 1.998867 seconds) securityworker stdout | 2025-11-04 09:06:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:45 GMT)" (scheduled at 2025-11-04 09:06:15.479769+00:00) securityworker stdout | 2025-11-04 09:06:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:15,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:15,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 15, 483946), 1, 2]) securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 15, 483946), 1, 2]) securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:21.530064+00:00 (in 5.994547 seconds) notificationworker stdout | 2025-11-04 09:06:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:15 GMT)" (scheduled at 2025-11-04 09:06:15.535095+00:00) notificationworker stdout | 2025-11-04 09:06:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:06:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:06:15,696 [286] [DEBUG] [app] Starting request: urn:request:61156f32-1767-443f-9460-8c05a6964344 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:15,697 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:15,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:15,701 [298] [DEBUG] [app] Starting request: urn:request:3b1daf10-3c01-4a12-bebe-00a95a82b64b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:15,701 [298] [DEBUG] [app] Ending request: urn:request:3b1daf10-3c01-4a12-bebe-00a95a82b64b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3b1daf10-3c01-4a12-bebe-00a95a82b64b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:15,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:15,701 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:15,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:15,704 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:15,705 [285] [DEBUG] [app] Starting request: urn:request:90a6a464-f959-497d-8cb7-15f335a60220 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:15,705 [285] [DEBUG] [app] Ending request: urn:request:90a6a464-f959-497d-8cb7-15f335a60220 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:90a6a464-f959-497d-8cb7-15f335a60220', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:15,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:15,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:15,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:15,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:15,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:15,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:15,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:15,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:15,730 [286] [DEBUG] [app] Ending request: urn:request:61156f32-1767-443f-9460-8c05a6964344 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:61156f32-1767-443f-9460-8c05a6964344', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:15,730 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:15,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:06:15,783 [285] [DEBUG] [app] Starting request: urn:request:10ae0af4-6903-4bb2-9685-11797ba0db1b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:15,788 [292] [DEBUG] [app] Starting request: urn:request:bfee1411-14ed-4cb9-af74-60ad6b19e5f2 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:15,788 [292] [DEBUG] [app] Ending request: urn:request:bfee1411-14ed-4cb9-af74-60ad6b19e5f2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bfee1411-14ed-4cb9-af74-60ad6b19e5f2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:15,792 [288] [DEBUG] [app] Starting request: urn:request:77d11662-4ba8-4ba4-af7f-bd825093726a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:15,792 [288] [DEBUG] [app] Ending request: urn:request:77d11662-4ba8-4ba4-af7f-bd825093726a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:77d11662-4ba8-4ba4-af7f-bd825093726a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:15,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:15,817 [285] [DEBUG] [app] Ending request: urn:request:10ae0af4-6903-4bb2-9685-11797ba0db1b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:10ae0af4-6903-4bb2-9685-11797ba0db1b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:06:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:27.122849+00:00 (in 9.999559 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:27 GMT)" (scheduled at 2025-11-04 09:06:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 17, 123591), True, datetime.datetime(2025, 11, 4, 9, 6, 17, 123591), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:06:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:45.479769+00:00 (in 28.000192 seconds) securityworker stdout | 2025-11-04 09:06:17,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:47 GMT)" (scheduled at 2025-11-04 09:06:17.479080+00:00) securityworker stdout | 2025-11-04 09:06:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:06:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:06:17,675 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:18,333 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:06:19,644 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:20,813 [288] [DEBUG] [app] Starting request: urn:request:59833d2b-0c47-45e1-808a-2a9cd34e0856 (/api/v1/user/notifications) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:06:20,813 [288] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:06:20,813 [288] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:06:20,813 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [endpoints.api] Checking permission for user whuquay gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:06:20,826 [288] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:06:20,827 [288] [DEBUG] [peewee] ('(SELECT "t1"."id", "t1"."uuid", "t1"."kind_id", "t1"."metadata_json", "t1"."dismissed", "t1"."lookup_path", "t1"."created", "t1"."created" AS "cd", "t1"."target_id" FROM "notification" AS "t1" INNER JOIN "notificationkind" AS "t2" ON ("t1"."kind_id" = "t2"."id") WHERE (("t1"."dismissed" = %s) AND ("t1"."target_id" = %s))) UNION (SELECT "t3"."id", "t3"."uuid", "t3"."kind_id", "t3"."metadata_json", "t3"."dismissed", "t3"."lookup_path", "t3"."created", "t3"."created" AS "cd", "t3"."target_id" FROM "notification" AS "t3" INNER JOIN "notificationkind" AS "t4" ON ("t3"."kind_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t5"."id" = "t3"."target_id") INNER JOIN "team" AS "t6" ON ("t5"."id" = "t6"."organization_id") INNER JOIN "teamrole" AS "t7" ON ("t6"."role_id" = "t7"."id") INNER JOIN "teammember" AS "t8" ON ("t6"."id" = "t8"."team_id") INNER JOIN "user" AS "t9" ON ("t8"."user_id" = "t9"."id") WHERE (("t3"."dismissed" = %s) AND (("t9"."id" = %s) AND ("t7"."name" = %s)))) ORDER BY cd desc LIMIT %s', [False, 1, False, 1, 'admin', 6]) gunicorn-web stdout | 2025-11-04 09:06:20,833 [288] [DEBUG] [app] Ending request: urn:request:59833d2b-0c47-45e1-808a-2a9cd34e0856 (/api/v1/user/notifications) {'endpoint': 'api.usernotificationlist', 'request_id': 'urn:request:59833d2b-0c47-45e1-808a-2a9cd34e0856', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/user/notifications', 'path': '/api/v1/user/notifications', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:06:20,834 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:20,834 [288] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:06:20 +0000] "GET /api/v1/user/notifications HTTP/1.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:06:20 +0000] "GET /api/v1/user/notifications HTTP/2.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.022 957 0.022) autopruneworker stdout | 2025-11-04 09:06:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:06:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:51.226294+00:00 (in 29.999555 seconds) autopruneworker stdout | 2025-11-04 09:06:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:51 GMT)" (scheduled at 2025-11-04 09:06:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:06:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247121234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:06:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:06:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:06:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:06:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:06:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:32.435813+00:00 (in 11.000941 seconds) namespacegcworker stdout | 2025-11-04 09:06:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:21 GMT)" (scheduled at 2025-11-04 09:06:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:06:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:06:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 21, 435160), True, datetime.datetime(2025, 11, 4, 9, 6, 21, 435160), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:06:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:06:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:31.530064+00:00 (in 9.999646 seconds) notificationworker stdout | 2025-11-04 09:06:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:31 GMT)" (scheduled at 2025-11-04 09:06:21.530064+00:00) notificationworker stdout | 2025-11-04 09:06:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 21, 530667), True, datetime.datetime(2025, 11, 4, 9, 6, 21, 530667), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:21,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:06:21,924 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:06:23,804 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:06:23,818 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:06:23,818 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:06:23,828 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:24,779 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:06:27,014 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:37.122849+00:00 (in 9.999584 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:37 GMT)" (scheduled at 2025-11-04 09:06:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 27, 123522), True, datetime.datetime(2025, 11, 4, 9, 6, 27, 123522), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:06:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:06:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:57.222846+00:00 (in 29.999656 seconds) buildlogsarchiver stdout | 2025-11-04 09:06:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:57 GMT)" (scheduled at 2025-11-04 09:06:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:06:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 6, 27, 223380), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:06:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:06:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:06:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:06:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:06:27,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:06:27,729 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:06:28,376 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:06:28,768 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:29,057 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:29,245 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:06:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:06:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:44.351216+00:00 (in 14.999581 seconds) quotatotalworker stdout | 2025-11-04 09:06:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:44 GMT)" (scheduled at 2025-11-04 09:06:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:06:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:06:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:06:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:06:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:06:29,372 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:29,399 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:29,466 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:30,697 [285] [DEBUG] [app] Starting request: urn:request:20efbbbf-1bf0-4d2e-a03a-9c80d826d00c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:30,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:30,701 [298] [DEBUG] [app] Starting request: urn:request:1ae086c4-243c-4188-abf6-d7351906d353 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:30,702 [298] [DEBUG] [app] Ending request: urn:request:1ae086c4-243c-4188-abf6-d7351906d353 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1ae086c4-243c-4188-abf6-d7351906d353', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:30,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:30,706 [288] [DEBUG] [app] Starting request: urn:request:6ee18928-ec65-4167-9dac-7b1b0eef08ff (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:30,706 [288] [DEBUG] [app] Ending request: urn:request:6ee18928-ec65-4167-9dac-7b1b0eef08ff (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6ee18928-ec65-4167-9dac-7b1b0eef08ff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:30,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:30,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:30,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:30,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:30,731 [285] [DEBUG] [app] Ending request: urn:request:20efbbbf-1bf0-4d2e-a03a-9c80d826d00c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:20efbbbf-1bf0-4d2e-a03a-9c80d826d00c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:30,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:06:30,784 [285] [DEBUG] [app] Starting request: urn:request:d5e163a7-65f7-4fd8-817c-4fef099729d7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:30,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:30,785 [298] [DEBUG] [app] Starting request: urn:request:1d17d875-2582-4837-b33b-ec6c2ebb1054 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:30,786 [298] [DEBUG] [app] Ending request: urn:request:1d17d875-2582-4837-b33b-ec6c2ebb1054 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1d17d875-2582-4837-b33b-ec6c2ebb1054', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:30,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:30,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:30,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:30,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:30,789 [288] [DEBUG] [app] Starting request: urn:request:b2ac0160-bd93-4c62-bf9f-0953f284f406 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:30,790 [288] [DEBUG] [app] Ending request: urn:request:b2ac0160-bd93-4c62-bf9f-0953f284f406 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b2ac0160-bd93-4c62-bf9f-0953f284f406', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:30,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:30,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:30,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:30,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:30,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:30,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:30,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:30,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:30,814 [285] [DEBUG] [app] Ending request: urn:request:d5e163a7-65f7-4fd8-817c-4fef099729d7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d5e163a7-65f7-4fd8-817c-4fef099729d7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:30,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:30,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:31,016 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:06:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:41.530064+00:00 (in 9.999581 seconds) notificationworker stdout | 2025-11-04 09:06:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:41 GMT)" (scheduled at 2025-11-04 09:06:31.530064+00:00) notificationworker stdout | 2025-11-04 09:06:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 31, 530768), True, datetime.datetime(2025, 11, 4, 9, 6, 31, 530768), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:06:31,806 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:06:32,005 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:06:32,005 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:06:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:21.434460+00:00 (in 48.998269 seconds) namespacegcworker stdout | 2025-11-04 09:06:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:32 GMT)" (scheduled at 2025-11-04 09:06:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:06:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:06:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:06:33,297 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:06:33,302 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:33,399 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:06:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:06:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:48.625192+00:00 (in 15.001281 seconds) exportactionlogsworker stdout | 2025-11-04 09:06:33,624 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:33 GMT)" (scheduled at 2025-11-04 09:06:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:06:33,624 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:06:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 33, 624226), True, datetime.datetime(2025, 11, 4, 9, 6, 33, 624226), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:06:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:06:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:06:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:33 GMT)" executed successfully builder stdout | 2025-11-04 09:06:33,829 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:06:33,842 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:06:33,842 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:06:33,852 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:06:34,187 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:06:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:38.437104+00:00 (in 3.000882 seconds) gcworker stdout | 2025-11-04 09:06:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:05 GMT)" (scheduled at 2025-11-04 09:06:35.435809+00:00) gcworker stdout | 2025-11-04 09:06:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037595436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:06:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:06:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:05 GMT)" executed successfully queuecleanupworker stdout | 2025-11-04 09:06:35,476 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:06:35,841 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:36,474 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:06:36,867 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:40.124445+00:00 (in 3.001170 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:47 GMT)" (scheduled at 2025-11-04 09:06:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 37, 123571), True, datetime.datetime(2025, 11, 4, 9, 6, 37, 123571), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:06:37,697 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,699 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,736 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,757 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,758 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,762 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,808 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,924 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:37,947 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:38,195 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:38,195 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:38,204 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:06:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:05.435809+00:00 (in 26.998300 seconds) gcworker stdout | 2025-11-04 09:06:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:08 GMT)" (scheduled at 2025-11-04 09:06:38.437104+00:00) gcworker stdout | 2025-11-04 09:06:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:06:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247138449, None, 1, 0]) gcworker stdout | 2025-11-04 09:06:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:06:39,038 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:06:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:06:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:01.922827+00:00 (in 21.997740 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:06:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:39 GMT)" (scheduled at 2025-11-04 09:06:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:06:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:06:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:06:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:47.122849+00:00 (in 6.997954 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:40 GMT)" (scheduled at 2025-11-04 09:06:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:06:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:06:40,809 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:06:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:51.530064+00:00 (in 9.999539 seconds) notificationworker stdout | 2025-11-04 09:06:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:51 GMT)" (scheduled at 2025-11-04 09:06:41.530064+00:00) notificationworker stdout | 2025-11-04 09:06:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 41, 530810), True, datetime.datetime(2025, 11, 4, 9, 6, 41, 530810), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:06:42,251 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:06:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:06:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:57.523915+00:00 (in 14.090667 seconds) repositorygcworker stdout | 2025-11-04 09:06:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:43 GMT)" (scheduled at 2025-11-04 09:06:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:06:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:06:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 43, 433548), True, datetime.datetime(2025, 11, 4, 9, 6, 43, 433548), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:06:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:06:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:06:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:43 GMT)" executed successfully builder stdout | 2025-11-04 09:06:43,852 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:06:43,865 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:06:43,865 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:06:43,875 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:06:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:06:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:59.351216+00:00 (in 14.999537 seconds) quotatotalworker stdout | 2025-11-04 09:06:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:59 GMT)" (scheduled at 2025-11-04 09:06:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:06:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:06:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:06:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:06:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:06:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:06:44,753 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:47.479080+00:00 (in 1.998904 seconds) securityworker stdout | 2025-11-04 09:06:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:15 GMT)" (scheduled at 2025-11-04 09:06:45.479769+00:00) securityworker stdout | 2025-11-04 09:06:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 45, 483626), 1, 2]) securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 45, 483626), 1, 2]) securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:06:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:15 GMT)" executed successfully securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker gunicorn-web stdout | 2025-11-04 09:06:45,696 [288] [DEBUG] [app] Starting request: urn:request:786b72dc-2726-4595-bf89-34d0cd76f920 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:45,701 [298] [DEBUG] [app] Starting request: urn:request:2fdf2cca-aac5-47f5-8f56-f8f631f7f42f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:45,701 [298] [DEBUG] [app] Ending request: urn:request:2fdf2cca-aac5-47f5-8f56-f8f631f7f42f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2fdf2cca-aac5-47f5-8f56-f8f631f7f42f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:45,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:06:45,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:45,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:45,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:45,705 [287] [DEBUG] [app] Starting request: urn:request:304608d9-4132-4f5c-bcb1-e6c6d760b55d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:45,705 [287] [DEBUG] [app] Ending request: urn:request:304608d9-4132-4f5c-bcb1-e6c6d760b55d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:304608d9-4132-4f5c-bcb1-e6c6d760b55d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:45,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:45,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:45,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:45,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:45,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:45,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:45,730 [288] [DEBUG] [app] Ending request: urn:request:786b72dc-2726-4595-bf89-34d0cd76f920 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:786b72dc-2726-4595-bf89-34d0cd76f920', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:45,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:45,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:06:45,783 [286] [DEBUG] [app] Starting request: urn:request:ab05408f-359a-4c55-8afa-1eab8a9b46ab (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:06:45,784 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:45,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:06:45,788 [299] [DEBUG] [app] Starting request: urn:request:62c6b0fd-0a83-4579-b6aa-744307293abf (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:06:45,788 [299] [DEBUG] [app] Ending request: urn:request:62c6b0fd-0a83-4579-b6aa-744307293abf (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:62c6b0fd-0a83-4579-b6aa-744307293abf', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:45,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:45,789 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:45,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:45,791 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:06:45,792 [285] [DEBUG] [app] Starting request: urn:request:34043759-10c5-499f-aff9-21b7919c3998 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:06:45,792 [285] [DEBUG] [app] Ending request: urn:request:34043759-10c5-499f-aff9-21b7919c3998 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:34043759-10c5-499f-aff9-21b7919c3998', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:06:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:06:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:45,793 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:45,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:45,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:45,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:45,801 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:45,801 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:45,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:45,814 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:45,817 [286] [DEBUG] [app] Ending request: urn:request:ab05408f-359a-4c55-8afa-1eab8a9b46ab (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ab05408f-359a-4c55-8afa-1eab8a9b46ab', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:45,817 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:06:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:06:45,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:06:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:06:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:57.122849+00:00 (in 9.999598 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:57 GMT)" (scheduled at 2025-11-04 09:06:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 47, 123563), True, datetime.datetime(2025, 11, 4, 9, 6, 47, 123563), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:47,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:47,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:06:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:15.479769+00:00 (in 28.000297 seconds) securityworker stdout | 2025-11-04 09:06:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:17 GMT)" (scheduled at 2025-11-04 09:06:47.479080+00:00) securityworker stdout | 2025-11-04 09:06:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:47,496 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:47,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:06:47,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:47,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:47,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:47,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 47, 483419), 1, 2]) securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:47,508 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 47, 483419), 1, 2]) securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:06:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:17 GMT)" executed successfully securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:06:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityscanningnotificationworker stdout | 2025-11-04 09:06:47,684 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:48,342 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:06:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:06:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:58.624878+00:00 (in 9.999298 seconds) exportactionlogsworker stdout | 2025-11-04 09:06:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:48 GMT)" (scheduled at 2025-11-04 09:06:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:06:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:06:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:06:49,666 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:06:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:06:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:21.226294+00:00 (in 29.999597 seconds) autopruneworker stdout | 2025-11-04 09:06:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:21 GMT)" (scheduled at 2025-11-04 09:06:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:06:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247151234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:06:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:06:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:06:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:01.530064+00:00 (in 9.999620 seconds) notificationworker stdout | 2025-11-04 09:06:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:01 GMT)" (scheduled at 2025-11-04 09:06:51.530064+00:00) notificationworker stdout | 2025-11-04 09:06:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 51, 530750), True, datetime.datetime(2025, 11, 4, 9, 6, 51, 530750), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:06:51,933 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:06:53,875 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:06:53,888 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:06:53,889 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:06:53,899 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:54,788 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:06:57,023 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:07.122849+00:00 (in 9.999578 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:07 GMT)" (scheduled at 2025-11-04 09:06:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 57, 123629), True, datetime.datetime(2025, 11, 4, 9, 6, 57, 123629), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:06:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:09.133645+00:00 (in 12.001449 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:57 GMT)" (scheduled at 2025-11-04 09:06:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:57,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:57,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:06:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:06:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:06:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:06:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:06:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:27.222846+00:00 (in 29.999598 seconds) buildlogsarchiver stdout | 2025-11-04 09:06:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:27 GMT)" (scheduled at 2025-11-04 09:06:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:06:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 6, 57, 223522), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:06:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:06:57,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:06:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:06:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:06:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:06:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:06:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:43.432867+00:00 (in 45.908557 seconds) repositorygcworker stdout | 2025-11-04 09:06:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:57 GMT)" (scheduled at 2025-11-04 09:06:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:06:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:06:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:06:57,738 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:06:58,385 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:06:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:06:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:33.622985+00:00 (in 34.997711 seconds) exportactionlogsworker stdout | 2025-11-04 09:06:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:11:58 GMT)" (scheduled at 2025-11-04 09:06:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:06:58,626 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 6, 58, 625558), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:06:58,637 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 6, 58, 625558), True, datetime.datetime(2025, 11, 4, 9, 6, 58, 625558), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:06:58,641 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 6, 58, 625558), True, datetime.datetime(2025, 11, 4, 9, 6, 58, 625558), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 6, 58, 625558), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:06:58,644 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:06:58,644 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:11:58 GMT)" executed successfully teamsyncworker stdout | 2025-11-04 09:06:58,777 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:59,067 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:59,256 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:06:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:06:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:14.351216+00:00 (in 14.999539 seconds) quotatotalworker stdout | 2025-11-04 09:06:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:14 GMT)" (scheduled at 2025-11-04 09:06:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:06:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:06:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:06:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:06:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:06:59,386 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:59,412 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:59,479 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:00,697 [288] [DEBUG] [app] Starting request: urn:request:95b4584e-e387-4f9f-be0e-cd7bc59cc9ba (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:00,701 [296] [DEBUG] [app] Starting request: urn:request:638c15a4-48ac-4924-83e3-826b361b4988 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:00,701 [296] [DEBUG] [app] Ending request: urn:request:638c15a4-48ac-4924-83e3-826b361b4988 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:638c15a4-48ac-4924-83e3-826b361b4988', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:07:00,701 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:00,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:00,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:00,705 [285] [DEBUG] [app] Starting request: urn:request:303d03fe-9ed5-4d9b-b00c-d8e49f2c4188 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:00,706 [285] [DEBUG] [app] Ending request: urn:request:303d03fe-9ed5-4d9b-b00c-d8e49f2c4188 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:303d03fe-9ed5-4d9b-b00c-d8e49f2c4188', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:00,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:00,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:00,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:00,730 [288] [DEBUG] [app] Ending request: urn:request:95b4584e-e387-4f9f-be0e-cd7bc59cc9ba (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:95b4584e-e387-4f9f-be0e-cd7bc59cc9ba', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:00,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:07:00,784 [285] [DEBUG] [app] Starting request: urn:request:3d2bffbb-31f5-4639-a340-31d3273b7b25 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:00,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:00,788 [292] [DEBUG] [app] Starting request: urn:request:7a9ab3ff-3f87-4aed-8216-d797520dec2c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:00,788 [292] [DEBUG] [app] Ending request: urn:request:7a9ab3ff-3f87-4aed-8216-d797520dec2c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7a9ab3ff-3f87-4aed-8216-d797520dec2c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:07:00,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:00,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:00,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:00,792 [287] [DEBUG] [app] Starting request: urn:request:77958ed7-b020-42c5-990c-3cce731780b5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:00,793 [287] [DEBUG] [app] Ending request: urn:request:77958ed7-b020-42c5-990c-3cce731780b5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:77958ed7-b020-42c5-990c-3cce731780b5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:00,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:00,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:00,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:00,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:00,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:00,817 [285] [DEBUG] [app] Ending request: urn:request:3d2bffbb-31f5-4639-a340-31d3273b7b25 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3d2bffbb-31f5-4639-a340-31d3273b7b25', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:00,818 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:00,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:01,025 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:07:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:11.530064+00:00 (in 9.999423 seconds) notificationworker stdout | 2025-11-04 09:07:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:11 GMT)" (scheduled at 2025-11-04 09:07:01.530064+00:00) notificationworker stdout | 2025-11-04 09:07:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 1, 530957), True, datetime.datetime(2025, 11, 4, 9, 7, 1, 530957), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:07:01,815 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:07:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:07:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:21.924304+00:00 (in 20.001048 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:07:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:01 GMT)" (scheduled at 2025-11-04 09:07:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:07:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:07:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 1, 923526), True, datetime.datetime(2025, 11, 4, 9, 7, 1, 923526), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:07:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:07:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:07:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:01 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:07:02,013 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:07:02,014 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:07:03,306 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:07:03,311 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:07:03,409 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:07:03,899 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:07:03,913 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:07:03,913 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:07:03,923 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:07:04,196 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:08.437104+00:00 (in 3.000934 seconds) gcworker stdout | 2025-11-04 09:07:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:35 GMT)" (scheduled at 2025-11-04 09:07:05.435809+00:00) gcworker stdout | 2025-11-04 09:07:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037625436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:07:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:07:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:35 GMT)" executed successfully queuecleanupworker stdout | 2025-11-04 09:07:05,485 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:07:05,849 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:06,483 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:07:06,875 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:17.122849+00:00 (in 9.999566 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:17 GMT)" (scheduled at 2025-11-04 09:07:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 7, 123590), True, datetime.datetime(2025, 11, 4, 9, 7, 7, 123590), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:07:07,712 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,711 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,751 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,769 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,774 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,771 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,822 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,940 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:07,958 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:08,205 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:08,206 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:08,225 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:35.435809+00:00 (in 26.998269 seconds) gcworker stdout | 2025-11-04 09:07:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:38 GMT)" (scheduled at 2025-11-04 09:07:08.437104+00:00) gcworker stdout | 2025-11-04 09:07:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:07:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247168449, None, 1, 0]) gcworker stdout | 2025-11-04 09:07:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:07:09,047 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:07:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:07:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:57.131795+00:00 (in 47.997722 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:09 GMT)" (scheduled at 2025-11-04 09:07:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:07:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:07:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:07:10,818 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:07:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:15.535095+00:00 (in 4.004577 seconds) notificationworker stdout | 2025-11-04 09:07:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:21 GMT)" (scheduled at 2025-11-04 09:07:11.530064+00:00) notificationworker stdout | 2025-11-04 09:07:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 11, 530767), True, datetime.datetime(2025, 11, 4, 9, 7, 11, 530767), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:07:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:07:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:12.222828+00:00 (in 59.999527 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:07:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:12 GMT)" (scheduled at 2025-11-04 09:07:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:07:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:07:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:07:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:07:12,260 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:07:13,923 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:07:13,937 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:07:13,937 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:07:13,947 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:07:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:07:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:29.351216+00:00 (in 14.999595 seconds) quotatotalworker stdout | 2025-11-04 09:07:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:29 GMT)" (scheduled at 2025-11-04 09:07:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:07:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:07:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:07:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:07:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:07:14,762 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:17.479080+00:00 (in 1.998842 seconds) securityworker stdout | 2025-11-04 09:07:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:45 GMT)" (scheduled at 2025-11-04 09:07:15.479769+00:00) securityworker stdout | 2025-11-04 09:07:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:07:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 15, 483764), 1, 2]) securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 15, 483764), 1, 2]) securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:07:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:45 GMT)" executed successfully securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker notificationworker stdout | 2025-11-04 09:07:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:21.530064+00:00 (in 5.994542 seconds) notificationworker stdout | 2025-11-04 09:07:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:15 GMT)" (scheduled at 2025-11-04 09:07:15.535095+00:00) notificationworker stdout | 2025-11-04 09:07:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:07:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:07:15,696 [288] [DEBUG] [app] Starting request: urn:request:18376cc5-dec6-4fdd-963c-d6a86bfc091e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:15,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:15,701 [292] [DEBUG] [app] Starting request: urn:request:cb4e6a94-7f7e-4107-9c19-843a01db0315 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:15,701 [292] [DEBUG] [app] Ending request: urn:request:cb4e6a94-7f7e-4107-9c19-843a01db0315 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cb4e6a94-7f7e-4107-9c19-843a01db0315', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:07:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:15,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:15,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:15,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:15,705 [285] [DEBUG] [app] Starting request: urn:request:0457664c-c9dc-426c-9a7f-23f63fc2ce31 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:15,706 [285] [DEBUG] [app] Ending request: urn:request:0457664c-c9dc-426c-9a7f-23f63fc2ce31 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0457664c-c9dc-426c-9a7f-23f63fc2ce31', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:15,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:15,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:15,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:15,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:15,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:15,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:15,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:15,731 [288] [DEBUG] [app] Ending request: urn:request:18376cc5-dec6-4fdd-963c-d6a86bfc091e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:18376cc5-dec6-4fdd-963c-d6a86bfc091e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:15,731 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:07:15,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:07:15,783 [285] [DEBUG] [app] Starting request: urn:request:d43452a8-9d3c-4d58-bf36-ee8fe687f34c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:15,786 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:15,787 [292] [DEBUG] [app] Starting request: urn:request:b865f8a7-a616-4dbe-a102-38794964921a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:15,787 [292] [DEBUG] [app] Ending request: urn:request:b865f8a7-a616-4dbe-a102-38794964921a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b865f8a7-a616-4dbe-a102-38794964921a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:07:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:15,791 [288] [DEBUG] [app] Starting request: urn:request:059e6be8-e67f-47b4-a7e9-d035313d246d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:15,792 [288] [DEBUG] [app] Ending request: urn:request:059e6be8-e67f-47b4-a7e9-d035313d246d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:059e6be8-e67f-47b4-a7e9-d035313d246d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:15,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:15,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:15,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:15,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:15,792 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:15,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:15,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:15,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:15,816 [285] [DEBUG] [app] Ending request: urn:request:d43452a8-9d3c-4d58-bf36-ee8fe687f34c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d43452a8-9d3c-4d58-bf36-ee8fe687f34c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:15,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:15,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:07:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:27.122849+00:00 (in 9.999611 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:27 GMT)" (scheduled at 2025-11-04 09:07:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 17, 123510), True, datetime.datetime(2025, 11, 4, 9, 7, 17, 123510), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:07:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:45.479769+00:00 (in 28.000205 seconds) securityworker stdout | 2025-11-04 09:07:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:47 GMT)" (scheduled at 2025-11-04 09:07:17.479080+00:00) securityworker stdout | 2025-11-04 09:07:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:17,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:17,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:07:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:07:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:07:17,704 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:18,352 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:07:19,675 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:07:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:07:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:51.226294+00:00 (in 29.999559 seconds) autopruneworker stdout | 2025-11-04 09:07:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:51 GMT)" (scheduled at 2025-11-04 09:07:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:07:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247181234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:07:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:07:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:07:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:07:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:07:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:32.435813+00:00 (in 11.000976 seconds) namespacegcworker stdout | 2025-11-04 09:07:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:21 GMT)" (scheduled at 2025-11-04 09:07:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:07:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:07:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 21, 435109), True, datetime.datetime(2025, 11, 4, 9, 7, 21, 435109), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:07:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:07:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:07:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:07:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:31.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:07:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:31 GMT)" (scheduled at 2025-11-04 09:07:21.530064+00:00) notificationworker stdout | 2025-11-04 09:07:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 21, 530726), True, datetime.datetime(2025, 11, 4, 9, 7, 21, 530726), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:31 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:07:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:07:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:39.924652+00:00 (in 17.999861 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:07:21,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:21 GMT)" (scheduled at 2025-11-04 09:07:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:07:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 7, 21, 925154), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:07:21,937 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 21, 925154), True, datetime.datetime(2025, 11, 4, 9, 7, 21, 925154), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:07:21,941 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 21, 925154), True, datetime.datetime(2025, 11, 4, 9, 7, 21, 925154), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 7, 21, 925154), 'secscanv4/%']) autopruneworker stdout | 2025-11-04 09:07:21,942 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:07:21,944 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:07:21,944 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:21 GMT)" executed successfully builder stdout | 2025-11-04 09:07:23,947 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:07:23,960 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:07:23,960 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:07:23,970 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:24,797 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:07:27,032 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:37.122849+00:00 (in 9.999601 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:37 GMT)" (scheduled at 2025-11-04 09:07:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 27, 123502), True, datetime.datetime(2025, 11, 4, 9, 7, 27, 123502), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:07:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:07:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:57.222846+00:00 (in 29.999579 seconds) buildlogsarchiver stdout | 2025-11-04 09:07:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:57 GMT)" (scheduled at 2025-11-04 09:07:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:07:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 7, 27, 223507), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:07:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:07:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:07:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:07:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:07:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:07:27,747 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:07:28,394 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:07:28,785 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:29,079 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:29,269 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:07:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:07:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:44.351216+00:00 (in 14.999594 seconds) quotatotalworker stdout | 2025-11-04 09:07:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:44 GMT)" (scheduled at 2025-11-04 09:07:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:07:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:07:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:07:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:07:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:07:29,401 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:29,426 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:29,493 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:30,697 [288] [DEBUG] [app] Starting request: urn:request:1215f0bd-8501-44a1-8ad2-4958ed8a4c08 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:30,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:30,701 [292] [DEBUG] [app] Starting request: urn:request:85b85db6-7c9c-40ee-bcb7-4cf2a821f1fb (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:30,702 [292] [DEBUG] [app] Ending request: urn:request:85b85db6-7c9c-40ee-bcb7-4cf2a821f1fb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:85b85db6-7c9c-40ee-bcb7-4cf2a821f1fb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:07:30,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:30,706 [285] [DEBUG] [app] Starting request: urn:request:60135df2-331d-44e2-9f2d-4977170f914b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:30,706 [285] [DEBUG] [app] Ending request: urn:request:60135df2-331d-44e2-9f2d-4977170f914b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:60135df2-331d-44e2-9f2d-4977170f914b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:30,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:30,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:30,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:30,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:30,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:30,730 [288] [DEBUG] [app] Ending request: urn:request:1215f0bd-8501-44a1-8ad2-4958ed8a4c08 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1215f0bd-8501-44a1-8ad2-4958ed8a4c08', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:30,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:07:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:07:30,784 [285] [DEBUG] [app] Starting request: urn:request:53972b4a-d276-4aea-a3d5-c7fad0dea519 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:30,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:30,788 [292] [DEBUG] [app] Starting request: urn:request:0b007229-df0e-40b3-8583-2067a63aba2b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:30,788 [292] [DEBUG] [app] Ending request: urn:request:0b007229-df0e-40b3-8583-2067a63aba2b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0b007229-df0e-40b3-8583-2067a63aba2b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:07:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:30,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:30,792 [287] [DEBUG] [app] Starting request: urn:request:d69de4a5-a0fa-49b2-bf35-013ca3bf478a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:30,792 [287] [DEBUG] [app] Ending request: urn:request:d69de4a5-a0fa-49b2-bf35-013ca3bf478a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d69de4a5-a0fa-49b2-bf35-013ca3bf478a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:30,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:30,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:30,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:30,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:30,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:30,817 [285] [DEBUG] [app] Ending request: urn:request:53972b4a-d276-4aea-a3d5-c7fad0dea519 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:53972b4a-d276-4aea-a3d5-c7fad0dea519', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:30,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:31,035 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:07:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:41.530064+00:00 (in 9.999531 seconds) notificationworker stdout | 2025-11-04 09:07:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:41 GMT)" (scheduled at 2025-11-04 09:07:31.530064+00:00) notificationworker stdout | 2025-11-04 09:07:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 31, 530790), True, datetime.datetime(2025, 11, 4, 9, 7, 31, 530790), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:07:31,824 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:07:32,022 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:07:32,023 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:07:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:07:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:08.435432+00:00 (in 35.999200 seconds) namespacegcworker stdout | 2025-11-04 09:07:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:32 GMT)" (scheduled at 2025-11-04 09:07:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:07:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:07:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:07:33,315 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:07:33,319 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:07:33,430 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:07:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:07:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:48.625192+00:00 (in 15.001813 seconds) exportactionlogsworker stdout | 2025-11-04 09:07:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:33 GMT)" (scheduled at 2025-11-04 09:07:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:07:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:07:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 33, 623668), True, datetime.datetime(2025, 11, 4, 9, 7, 33, 623668), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:07:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:07:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:07:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:33 GMT)" executed successfully builder stdout | 2025-11-04 09:07:33,970 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:07:33,984 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:07:33,984 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:07:33,994 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:07:34,205 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:38.437104+00:00 (in 3.000914 seconds) gcworker stdout | 2025-11-04 09:07:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:05 GMT)" (scheduled at 2025-11-04 09:07:35.435809+00:00) gcworker stdout | 2025-11-04 09:07:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037655436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:07:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:07:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:05 GMT)" executed successfully queuecleanupworker stdout | 2025-11-04 09:07:35,494 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:07:35,859 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:36,493 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:07:36,884 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:40.124445+00:00 (in 3.001191 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:47 GMT)" (scheduled at 2025-11-04 09:07:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 37, 123538), True, datetime.datetime(2025, 11, 4, 9, 7, 37, 123538), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:07:37,725 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,736 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,765 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,780 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,785 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,807 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,835 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,954 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:37,980 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:38,216 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:38,216 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:38,235 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:05.435809+00:00 (in 26.998331 seconds) gcworker stdout | 2025-11-04 09:07:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:08 GMT)" (scheduled at 2025-11-04 09:07:38.437104+00:00) gcworker stdout | 2025-11-04 09:07:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:07:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247198449, None, 1, 0]) gcworker stdout | 2025-11-04 09:07:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:07:39,056 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:07:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:07:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:01.922827+00:00 (in 21.997776 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:07:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:39 GMT)" (scheduled at 2025-11-04 09:07:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:07:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:07:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:07:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:47.122849+00:00 (in 6.997966 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:40 GMT)" (scheduled at 2025-11-04 09:07:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:07:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:07:40,826 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:07:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:51.530064+00:00 (in 9.999515 seconds) notificationworker stdout | 2025-11-04 09:07:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:51 GMT)" (scheduled at 2025-11-04 09:07:41.530064+00:00) notificationworker stdout | 2025-11-04 09:07:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 41, 530890), True, datetime.datetime(2025, 11, 4, 9, 7, 41, 530890), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:07:42,268 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:07:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:07:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:57.523915+00:00 (in 14.090630 seconds) repositorygcworker stdout | 2025-11-04 09:07:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:43 GMT)" (scheduled at 2025-11-04 09:07:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:07:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:07:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 43, 433576), True, datetime.datetime(2025, 11, 4, 9, 7, 43, 433576), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:07:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:07:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:07:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:43 GMT)" executed successfully builder stdout | 2025-11-04 09:07:43,994 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:07:44,008 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:07:44,008 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:07:44,018 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:07:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:07:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:59.351216+00:00 (in 14.999555 seconds) quotatotalworker stdout | 2025-11-04 09:07:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:59 GMT)" (scheduled at 2025-11-04 09:07:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:07:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:07:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:07:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:07:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:07:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:07:44,771 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:47.479080+00:00 (in 1.998910 seconds) securityworker stdout | 2025-11-04 09:07:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:15 GMT)" (scheduled at 2025-11-04 09:07:45.479769+00:00) securityworker stdout | 2025-11-04 09:07:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 45, 483592), 1, 2]) securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 45, 483592), 1, 2]) securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:07:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:07:45,696 [288] [DEBUG] [app] Starting request: urn:request:30526b8d-140a-4754-9359-d2cfff513cd2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:45,701 [296] [DEBUG] [app] Starting request: urn:request:759e6e68-8388-45d2-add7-734337c7b009 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:45,701 [296] [DEBUG] [app] Ending request: urn:request:759e6e68-8388-45d2-add7-734337c7b009 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:759e6e68-8388-45d2-add7-734337c7b009', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:07:45,701 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:45,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:45,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:45,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:45,705 [287] [DEBUG] [app] Starting request: urn:request:968f8f49-6b1f-4727-8110-2afeb10b9a10 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:45,705 [287] [DEBUG] [app] Ending request: urn:request:968f8f49-6b1f-4727-8110-2afeb10b9a10 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:968f8f49-6b1f-4727-8110-2afeb10b9a10', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:45,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:45,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:45,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:45,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:45,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:45,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:45,730 [288] [DEBUG] [app] Ending request: urn:request:30526b8d-140a-4754-9359-d2cfff513cd2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:30526b8d-140a-4754-9359-d2cfff513cd2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:45,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:45,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:07:45,784 [285] [DEBUG] [app] Starting request: urn:request:13e74127-7dc8-4601-b253-75665b96ada7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:07:45,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:45,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:07:45,788 [292] [DEBUG] [app] Starting request: urn:request:e31fc62f-5110-45a4-a2f7-62df858c56ad (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:07:45,789 [292] [DEBUG] [app] Ending request: urn:request:e31fc62f-5110-45a4-a2f7-62df858c56ad (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e31fc62f-5110-45a4-a2f7-62df858c56ad', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:07:45,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:45,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:45,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:07:45,793 [288] [DEBUG] [app] Starting request: urn:request:a930962a-c03b-4aa6-bad3-a5e26ba7c626 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:07:45,793 [288] [DEBUG] [app] Ending request: urn:request:a930962a-c03b-4aa6-bad3-a5e26ba7c626 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a930962a-c03b-4aa6-bad3-a5e26ba7c626', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:45,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:07:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:07:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:45,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:45,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:45,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:45,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:45,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:45,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:45,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:45,817 [285] [DEBUG] [app] Ending request: urn:request:13e74127-7dc8-4601-b253-75665b96ada7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:13e74127-7dc8-4601-b253-75665b96ada7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:45,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:45,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:07:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:07:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:07:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:57.122849+00:00 (in 9.999596 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:57 GMT)" (scheduled at 2025-11-04 09:07:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 47, 123543), True, datetime.datetime(2025, 11, 4, 9, 7, 47, 123543), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:07:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:15.479769+00:00 (in 28.000240 seconds) securityworker stdout | 2025-11-04 09:07:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:17 GMT)" (scheduled at 2025-11-04 09:07:47.479080+00:00) securityworker stdout | 2025-11-04 09:07:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:07:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 47, 483629), 1, 2]) securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 47, 483629), 1, 2]) securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:07:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:07:47,713 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:48,361 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:07:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:07:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:33.622985+00:00 (in 44.997343 seconds) exportactionlogsworker stdout | 2025-11-04 09:07:48,626 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:48 GMT)" (scheduled at 2025-11-04 09:07:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:07:48,626 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:07:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:07:49,684 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:07:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:07:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:21.226294+00:00 (in 29.999553 seconds) autopruneworker stdout | 2025-11-04 09:07:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:21 GMT)" (scheduled at 2025-11-04 09:07:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:07:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247211234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:07:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:07:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:07:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:07:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:01.530064+00:00 (in 9.999544 seconds) notificationworker stdout | 2025-11-04 09:07:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:01 GMT)" (scheduled at 2025-11-04 09:07:51.530064+00:00) notificationworker stdout | 2025-11-04 09:07:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 51, 530792), True, datetime.datetime(2025, 11, 4, 9, 7, 51, 530792), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:07:51,950 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:07:54,018 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:07:54,032 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:07:54,032 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:07:54,042 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:54,815 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:07:57,040 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:07.122849+00:00 (in 9.999590 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:07 GMT)" (scheduled at 2025-11-04 09:07:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 57, 123533), True, datetime.datetime(2025, 11, 4, 9, 7, 57, 123533), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:07:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:09.133645+00:00 (in 12.001422 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:57 GMT)" (scheduled at 2025-11-04 09:07:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:07:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:07:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:07:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:07:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:07:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:27.222846+00:00 (in 29.999605 seconds) buildlogsarchiver stdout | 2025-11-04 09:07:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:27 GMT)" (scheduled at 2025-11-04 09:07:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:07:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 7, 57, 223734), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:07:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:07:57,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:07:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:07:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:07:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:07:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:07:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:43.432867+00:00 (in 45.908537 seconds) repositorygcworker stdout | 2025-11-04 09:07:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:57 GMT)" (scheduled at 2025-11-04 09:07:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:07:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:07:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:07:57,756 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:07:58,403 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:07:58,794 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:59,089 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:59,281 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:07:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:07:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:14.351216+00:00 (in 14.999569 seconds) quotatotalworker stdout | 2025-11-04 09:07:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:14 GMT)" (scheduled at 2025-11-04 09:07:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:07:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:07:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:07:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:07:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:07:59,415 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:59,440 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:59,507 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:00,697 [287] [DEBUG] [app] Starting request: urn:request:0682cadf-e90c-4a73-ace2-97dec0ede98d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:00,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:00,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:00,701 [292] [DEBUG] [app] Starting request: urn:request:e5673530-521a-48e6-9f39-057ecae8a459 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:00,701 [292] [DEBUG] [app] Ending request: urn:request:e5673530-521a-48e6-9f39-057ecae8a459 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e5673530-521a-48e6-9f39-057ecae8a459', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:08:00,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:00,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:00,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:00,705 [286] [DEBUG] [app] Starting request: urn:request:62d02cdc-4271-48a1-b9a6-0abef1f58d65 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:00,706 [286] [DEBUG] [app] Ending request: urn:request:62d02cdc-4271-48a1-b9a6-0abef1f58d65 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:62d02cdc-4271-48a1-b9a6-0abef1f58d65', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:00,706 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:00,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:00,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:00,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:00,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:00,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:00,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:00,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:00,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:00,730 [287] [DEBUG] [app] Ending request: urn:request:0682cadf-e90c-4a73-ace2-97dec0ede98d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0682cadf-e90c-4a73-ace2-97dec0ede98d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:00,730 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:08:00,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:08:00,784 [285] [DEBUG] [app] Starting request: urn:request:77a8c89a-589d-4a6b-a9ca-40de3079450c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:00,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:00,788 [292] [DEBUG] [app] Starting request: urn:request:572933c0-1453-45a9-b70a-e3cd195d2302 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:00,788 [292] [DEBUG] [app] Ending request: urn:request:572933c0-1453-45a9-b70a-e3cd195d2302 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:572933c0-1453-45a9-b70a-e3cd195d2302', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:00,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:08:00,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:00,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:00,792 [288] [DEBUG] [app] Starting request: urn:request:dcee0ac4-74fb-42c3-ac20-42bbe7a95570 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:00,793 [288] [DEBUG] [app] Ending request: urn:request:dcee0ac4-74fb-42c3-ac20-42bbe7a95570 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dcee0ac4-74fb-42c3-ac20-42bbe7a95570', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:00,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:00,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:00,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:00,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:00,817 [285] [DEBUG] [app] Ending request: urn:request:77a8c89a-589d-4a6b-a9ca-40de3079450c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:77a8c89a-589d-4a6b-a9ca-40de3079450c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:00,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:01,044 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:08:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:11.530064+00:00 (in 9.999540 seconds) notificationworker stdout | 2025-11-04 09:08:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:11 GMT)" (scheduled at 2025-11-04 09:08:01.530064+00:00) notificationworker stdout | 2025-11-04 09:08:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 1, 530803), True, datetime.datetime(2025, 11, 4, 9, 8, 1, 530803), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:08:01,832 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:08:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:08:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:39.924652+00:00 (in 38.001450 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:08:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:01 GMT)" (scheduled at 2025-11-04 09:08:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:08:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:08:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 1, 923475), True, datetime.datetime(2025, 11, 4, 9, 8, 1, 923475), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:08:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:08:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:08:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:01 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:08:02,030 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:08:02,031 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:08:03,324 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:08:03,328 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:08:03,439 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:08:04,042 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:08:04,055 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:08:04,055 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:08:04,065 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:08:04,215 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:08.437104+00:00 (in 3.000911 seconds) gcworker stdout | 2025-11-04 09:08:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:35 GMT)" (scheduled at 2025-11-04 09:08:05.435809+00:00) gcworker stdout | 2025-11-04 09:08:05,436 [72] [DEBUG] [peewee] ('SELECT DISTINCT "t1"."removed_tag_expiration_s" FROM "user" AS "t1" LIMIT %s', [100]) pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:05,448 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037685448, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:08:05,453 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:08:05,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:05,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:35 GMT)" executed successfully queuecleanupworker stdout | 2025-11-04 09:08:05,503 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:08:05,868 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:08:06,503 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:08:06,893 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:17.122849+00:00 (in 9.999577 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:17 GMT)" (scheduled at 2025-11-04 09:08:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 7, 123588), True, datetime.datetime(2025, 11, 4, 9, 8, 7, 123588), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:08:07,736 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,749 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,778 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,795 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,796 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,818 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,848 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,969 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:07,993 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:08,227 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:08,227 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:08,245 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:08:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:08:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:21.434460+00:00 (in 12.998623 seconds) namespacegcworker stdout | 2025-11-04 09:08:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:13:08 GMT)" (scheduled at 2025-11-04 09:08:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:08:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 8, 8, 436069), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:08:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:35.435809+00:00 (in 26.998347 seconds) gcworker stdout | 2025-11-04 09:08:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:38 GMT)" (scheduled at 2025-11-04 09:08:08.437104+00:00) gcworker stdout | 2025-11-04 09:08:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:08:08,448 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247228448, None, 1, 0]) namespacegcworker stdout | 2025-11-04 09:08:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 8, 8, 436069), True, datetime.datetime(2025, 11, 4, 9, 8, 8, 436069), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:08:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:08,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:08:08,452 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 8, 8, 436069), True, datetime.datetime(2025, 11, 4, 9, 8, 8, 436069), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 8, 8, 436069), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:08:08,455 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:08:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:13:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:08:09,065 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:08:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:08:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:57.131795+00:00 (in 47.997734 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:09 GMT)" (scheduled at 2025-11-04 09:08:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:08:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:08:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:08:10,835 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:08:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:15.535095+00:00 (in 4.004588 seconds) notificationworker stdout | 2025-11-04 09:08:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:21 GMT)" (scheduled at 2025-11-04 09:08:11.530064+00:00) notificationworker stdout | 2025-11-04 09:08:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 11, 530814), True, datetime.datetime(2025, 11, 4, 9, 8, 11, 530814), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:08:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:08:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:12.222828+00:00 (in 59.999525 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:08:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:12 GMT)" (scheduled at 2025-11-04 09:08:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:08:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:08:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:08:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:08:12,277 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:08:14,065 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:08:14,078 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:08:14,078 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:08:14,089 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:08:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:08:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:29.351216+00:00 (in 14.999546 seconds) quotatotalworker stdout | 2025-11-04 09:08:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:29 GMT)" (scheduled at 2025-11-04 09:08:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:08:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:08:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:08:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:08:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:08:14,780 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:08:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:17.479080+00:00 (in 1.998851 seconds) securityworker stdout | 2025-11-04 09:08:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:45 GMT)" (scheduled at 2025-11-04 09:08:15.479769+00:00) securityworker stdout | 2025-11-04 09:08:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:08:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:08:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 15, 483731), 1, 2]) securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 15, 483731), 1, 2]) securityworker stdout | 2025-11-04 09:08:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:08:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:45 GMT)" executed successfully securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker notificationworker stdout | 2025-11-04 09:08:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:21.530064+00:00 (in 5.994495 seconds) notificationworker stdout | 2025-11-04 09:08:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:15 GMT)" (scheduled at 2025-11-04 09:08:15.535095+00:00) notificationworker stdout | 2025-11-04 09:08:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:08:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:08:15,697 [285] [DEBUG] [app] Starting request: urn:request:e0b81d2c-9c95-4e41-b8cd-54f53ab4f600 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:15,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:15,701 [296] [DEBUG] [app] Starting request: urn:request:2e827e09-2dc5-45f8-8018-31b559be7bb9 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:15,702 [296] [DEBUG] [app] Ending request: urn:request:2e827e09-2dc5-45f8-8018-31b559be7bb9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2e827e09-2dc5-45f8-8018-31b559be7bb9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:08:15,702 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:15,706 [288] [DEBUG] [app] Starting request: urn:request:9371b01c-7626-4011-90f1-1601d91e0347 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:15,706 [288] [DEBUG] [app] Ending request: urn:request:9371b01c-7626-4011-90f1-1601d91e0347 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9371b01c-7626-4011-90f1-1601d91e0347', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:15,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:15,731 [285] [DEBUG] [app] Ending request: urn:request:e0b81d2c-9c95-4e41-b8cd-54f53ab4f600 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e0b81d2c-9c95-4e41-b8cd-54f53ab4f600', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:15,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:15,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:08:15,783 [285] [DEBUG] [app] Starting request: urn:request:268073b8-3778-48f5-953c-23ce284fc12f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:15,785 [292] [DEBUG] [app] Starting request: urn:request:4c865c1b-1377-489a-9ab6-2093de738990 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:15,785 [292] [DEBUG] [app] Ending request: urn:request:4c865c1b-1377-489a-9ab6-2093de738990 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:4c865c1b-1377-489a-9ab6-2093de738990', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:15,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:08:15,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:15,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:15,789 [288] [DEBUG] [app] Starting request: urn:request:e66b62e8-41de-4ffb-9f9d-94cfc0bd1cd1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:15,789 [288] [DEBUG] [app] Ending request: urn:request:e66b62e8-41de-4ffb-9f9d-94cfc0bd1cd1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e66b62e8-41de-4ffb-9f9d-94cfc0bd1cd1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:15,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:15,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:15,813 [285] [DEBUG] [app] Ending request: urn:request:268073b8-3778-48f5-953c-23ce284fc12f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:268073b8-3778-48f5-953c-23ce284fc12f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:15,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:08:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:27.122849+00:00 (in 9.999594 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:27 GMT)" (scheduled at 2025-11-04 09:08:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 17, 123518), True, datetime.datetime(2025, 11, 4, 9, 8, 17, 123518), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:08:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:45.479769+00:00 (in 28.000228 seconds) securityworker stdout | 2025-11-04 09:08:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:47 GMT)" (scheduled at 2025-11-04 09:08:17.479080+00:00) securityworker stdout | 2025-11-04 09:08:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:08:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:08:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:08:17,722 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:18,380 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:08:19,694 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:08:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:08:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:51.226294+00:00 (in 29.999538 seconds) autopruneworker stdout | 2025-11-04 09:08:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:51 GMT)" (scheduled at 2025-11-04 09:08:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:08:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247241235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:08:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:08:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:08:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:08:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:08:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:32.435813+00:00 (in 11.000944 seconds) namespacegcworker stdout | 2025-11-04 09:08:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:21 GMT)" (scheduled at 2025-11-04 09:08:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:08:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:08:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 21, 435130), True, datetime.datetime(2025, 11, 4, 9, 8, 21, 435130), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:08:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:08:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:08:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:08:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:31.530064+00:00 (in 9.999556 seconds) notificationworker stdout | 2025-11-04 09:08:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:31 GMT)" (scheduled at 2025-11-04 09:08:21.530064+00:00) notificationworker stdout | 2025-11-04 09:08:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 21, 530806), True, datetime.datetime(2025, 11, 4, 9, 8, 21, 530806), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:08:21,959 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:08:24,089 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:08:24,102 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:08:24,102 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:08:24,112 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:24,823 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:08:27,049 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:37.122849+00:00 (in 9.999588 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:37 GMT)" (scheduled at 2025-11-04 09:08:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 27, 123532), True, datetime.datetime(2025, 11, 4, 9, 8, 27, 123532), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:08:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:08:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:57.222846+00:00 (in 29.999585 seconds) buildlogsarchiver stdout | 2025-11-04 09:08:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:57 GMT)" (scheduled at 2025-11-04 09:08:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:08:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 8, 27, 223511), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:08:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:08:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:08:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:08:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:08:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:08:27,765 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:08:28,412 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:08:28,803 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:29,100 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:29,298 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:08:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:08:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:44.351216+00:00 (in 14.999572 seconds) quotatotalworker stdout | 2025-11-04 09:08:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:44 GMT)" (scheduled at 2025-11-04 09:08:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:08:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:08:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:08:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:08:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:08:29,429 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:29,454 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:29,521 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:30,697 [285] [DEBUG] [app] Starting request: urn:request:89d79e9d-9f0e-41d1-a4d9-963250024d5c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:30,701 [292] [DEBUG] [app] Starting request: urn:request:10126b6e-e900-40aa-8ef2-9ae89ef40532 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:30,702 [292] [DEBUG] [app] Ending request: urn:request:10126b6e-e900-40aa-8ef2-9ae89ef40532 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:10126b6e-e900-40aa-8ef2-9ae89ef40532', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:08:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:30,706 [287] [DEBUG] [app] Starting request: urn:request:3498af9c-686f-4131-b202-c63da8839e7b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:30,706 [287] [DEBUG] [app] Ending request: urn:request:3498af9c-686f-4131-b202-c63da8839e7b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3498af9c-686f-4131-b202-c63da8839e7b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:30,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:30,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:30,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:30,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:30,731 [285] [DEBUG] [app] Ending request: urn:request:89d79e9d-9f0e-41d1-a4d9-963250024d5c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:89d79e9d-9f0e-41d1-a4d9-963250024d5c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:30,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:30,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) gunicorn-web stdout | 2025-11-04 09:08:30,783 [285] [DEBUG] [app] Starting request: urn:request:d76bf3f6-5a54-45d8-bfaf-dc2d0b1f717a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:30,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:30,785 [296] [DEBUG] [app] Starting request: urn:request:cb240fe8-3c72-4a32-830a-97ebc00088ca (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:30,785 [296] [DEBUG] [app] Ending request: urn:request:cb240fe8-3c72-4a32-830a-97ebc00088ca (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cb240fe8-3c72-4a32-830a-97ebc00088ca', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:30,785 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:08:30,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:30,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:30,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:30,789 [288] [DEBUG] [app] Starting request: urn:request:8c1a9236-8e1a-4cc5-9aee-c4fc7e2c7820 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:30,789 [288] [DEBUG] [app] Ending request: urn:request:8c1a9236-8e1a-4cc5-9aee-c4fc7e2c7820 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8c1a9236-8e1a-4cc5-9aee-c4fc7e2c7820', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:30,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:30,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:30,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:30,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:30,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:30,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:30,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:30,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:30,814 [285] [DEBUG] [app] Ending request: urn:request:d76bf3f6-5a54-45d8-bfaf-dc2d0b1f717a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d76bf3f6-5a54-45d8-bfaf-dc2d0b1f717a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:30,814 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) gunicorn-web stdout | 2025-11-04 09:08:30,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:08:31,053 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:08:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:41.530064+00:00 (in 9.999545 seconds) notificationworker stdout | 2025-11-04 09:08:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:41 GMT)" (scheduled at 2025-11-04 09:08:31.530064+00:00) notificationworker stdout | 2025-11-04 09:08:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 31, 530774), True, datetime.datetime(2025, 11, 4, 9, 8, 31, 530774), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:08:31,841 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:08:32,039 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:08:32,040 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:08:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:08:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:21.434460+00:00 (in 48.998234 seconds) namespacegcworker stdout | 2025-11-04 09:08:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:32 GMT)" (scheduled at 2025-11-04 09:08:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:08:32,437 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:08:32,437 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:08:33,333 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:08:33,336 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:08:33,448 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:08:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:08:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:48.625192+00:00 (in 15.001832 seconds) exportactionlogsworker stdout | 2025-11-04 09:08:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:33 GMT)" (scheduled at 2025-11-04 09:08:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:08:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:08:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 33, 623673), True, datetime.datetime(2025, 11, 4, 9, 8, 33, 623673), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:08:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:08:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:08:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:33 GMT)" executed successfully builder stdout | 2025-11-04 09:08:34,113 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:08:34,126 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:08:34,126 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:08:34,136 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:08:34,224 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:38.437104+00:00 (in 3.000892 seconds) gcworker stdout | 2025-11-04 09:08:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:05 GMT)" (scheduled at 2025-11-04 09:08:35.435809+00:00) gcworker stdout | 2025-11-04 09:08:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037715436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:08:35,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:08:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:08:35,523 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:08:35,890 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:08:36,513 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:08:36,901 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:40.124445+00:00 (in 3.001133 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:47 GMT)" (scheduled at 2025-11-04 09:08:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 37, 123593), True, datetime.datetime(2025, 11, 4, 9, 8, 37, 123593), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:37,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:37,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:08:37,746 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:37,762 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:37,788 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:37,805 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:37,809 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:37,830 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:37,860 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:37,989 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:38,008 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:38,237 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:38,237 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:38,255 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:05.435809+00:00 (in 26.998290 seconds) gcworker stdout | 2025-11-04 09:08:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:08 GMT)" (scheduled at 2025-11-04 09:08:38.437104+00:00) gcworker stdout | 2025-11-04 09:08:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:08:38,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247258449, None, 1, 0]) gcworker stdout | 2025-11-04 09:08:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:38,454 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:08:39,073 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:08:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:08:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:01.922827+00:00 (in 21.997739 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:08:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:39 GMT)" (scheduled at 2025-11-04 09:08:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:08:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:08:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:08:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:47.122849+00:00 (in 6.997978 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:40 GMT)" (scheduled at 2025-11-04 09:08:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:08:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:08:40,844 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:08:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:51.530064+00:00 (in 9.999610 seconds) notificationworker stdout | 2025-11-04 09:08:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:51 GMT)" (scheduled at 2025-11-04 09:08:41.530064+00:00) notificationworker stdout | 2025-11-04 09:08:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 41, 530741), True, datetime.datetime(2025, 11, 4, 9, 8, 41, 530741), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:08:42,286 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:08:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:08:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:57.523915+00:00 (in 14.090628 seconds) repositorygcworker stdout | 2025-11-04 09:08:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:43 GMT)" (scheduled at 2025-11-04 09:08:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:08:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:08:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 43, 433618), True, datetime.datetime(2025, 11, 4, 9, 8, 43, 433618), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:08:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:08:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:08:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:43 GMT)" executed successfully builder stdout | 2025-11-04 09:08:44,136 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:08:44,151 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:08:44,151 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:08:44,171 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:08:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:08:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:59.351216+00:00 (in 14.999588 seconds) quotatotalworker stdout | 2025-11-04 09:08:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:59 GMT)" (scheduled at 2025-11-04 09:08:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:08:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:08:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:08:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:08:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:08:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:08:44,789 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:08:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:47.479080+00:00 (in 1.998890 seconds) securityworker stdout | 2025-11-04 09:08:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:15 GMT)" (scheduled at 2025-11-04 09:08:45.479769+00:00) securityworker stdout | 2025-11-04 09:08:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 45, 483663), 1, 2]) securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 45, 483663), 1, 2]) securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:08:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:08:45,697 [285] [DEBUG] [app] Starting request: urn:request:ef26e47d-fb54-43ee-add0-80322c8818ca (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:45,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:45,702 [299] [DEBUG] [app] Starting request: urn:request:e94c8fbd-5570-4efe-afc9-f931f74ae3af (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:45,702 [299] [DEBUG] [app] Ending request: urn:request:e94c8fbd-5570-4efe-afc9-f931f74ae3af (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e94c8fbd-5570-4efe-afc9-f931f74ae3af', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:45,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:08:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:45,706 [288] [DEBUG] [app] Starting request: urn:request:0ac955a2-a08a-46f3-87ee-80989b339ee1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:45,706 [288] [DEBUG] [app] Ending request: urn:request:0ac955a2-a08a-46f3-87ee-80989b339ee1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0ac955a2-a08a-46f3-87ee-80989b339ee1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:45,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:45,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:45,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:45,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:45,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:45,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:45,731 [285] [DEBUG] [app] Ending request: urn:request:ef26e47d-fb54-43ee-add0-80322c8818ca (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ef26e47d-fb54-43ee-add0-80322c8818ca', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:45,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:45,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:08:45,784 [288] [DEBUG] [app] Starting request: urn:request:5552b168-bb5b-428c-9933-aaf9e9b79390 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:08:45,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:45,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:08:45,788 [292] [DEBUG] [app] Starting request: urn:request:731f74ad-e8c4-4d64-98c1-a4b34efe7c9b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:08:45,789 [292] [DEBUG] [app] Ending request: urn:request:731f74ad-e8c4-4d64-98c1-a4b34efe7c9b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:731f74ad-e8c4-4d64-98c1-a4b34efe7c9b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:08:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:45,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:45,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:45,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:08:45,793 [285] [DEBUG] [app] Starting request: urn:request:ca731665-db04-49c2-9eae-07e8439dddcb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:08:45,793 [285] [DEBUG] [app] Ending request: urn:request:ca731665-db04-49c2-9eae-07e8439dddcb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ca731665-db04-49c2-9eae-07e8439dddcb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:08:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:08:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:08:45,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:45,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:45,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:45,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:45,817 [288] [DEBUG] [app] Ending request: urn:request:5552b168-bb5b-428c-9933-aaf9e9b79390 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5552b168-bb5b-428c-9933-aaf9e9b79390', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:45,817 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:08:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:08:45,818 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:08:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:08:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:57.122849+00:00 (in 9.999568 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:57 GMT)" (scheduled at 2025-11-04 09:08:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 47, 123547), True, datetime.datetime(2025, 11, 4, 9, 8, 47, 123547), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:08:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:15.479769+00:00 (in 28.000199 seconds) securityworker stdout | 2025-11-04 09:08:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:17 GMT)" (scheduled at 2025-11-04 09:08:47.479080+00:00) securityworker stdout | 2025-11-04 09:08:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:08:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:08:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 47, 483084), 1, 2]) securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 47, 483084), 1, 2]) securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:08:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:08:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:08:47,731 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:48,390 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:08:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:08:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:33.622985+00:00 (in 44.997361 seconds) exportactionlogsworker stdout | 2025-11-04 09:08:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:48 GMT)" (scheduled at 2025-11-04 09:08:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:08:48,626 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:08:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:08:49,703 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:08:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:08:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:21.226294+00:00 (in 29.999557 seconds) autopruneworker stdout | 2025-11-04 09:08:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:21 GMT)" (scheduled at 2025-11-04 09:08:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:08:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247271235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:08:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:08:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:08:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:08:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:01.530064+00:00 (in 9.999584 seconds) notificationworker stdout | 2025-11-04 09:08:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:01 GMT)" (scheduled at 2025-11-04 09:08:51.530064+00:00) notificationworker stdout | 2025-11-04 09:08:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 51, 530762), True, datetime.datetime(2025, 11, 4, 9, 8, 51, 530762), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:51,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:51,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:51,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:08:51,967 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:08:54,171 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:08:54,185 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:08:54,185 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:08:54,195 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:54,833 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:08:57,058 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:07.122849+00:00 (in 9.999612 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:07 GMT)" (scheduled at 2025-11-04 09:08:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 57, 123527), True, datetime.datetime(2025, 11, 4, 9, 8, 57, 123527), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:08:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:09.133645+00:00 (in 12.001476 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:57 GMT)" (scheduled at 2025-11-04 09:08:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:08:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:08:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:08:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:08:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:08:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:27.222846+00:00 (in 29.999592 seconds) buildlogsarchiver stdout | 2025-11-04 09:08:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:27 GMT)" (scheduled at 2025-11-04 09:08:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:08:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 8, 57, 223477), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:08:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:08:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:08:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:08:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:08:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:08:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:08:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:02.523412+00:00 (in 4.999060 seconds) repositorygcworker stdout | 2025-11-04 09:08:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:57 GMT)" (scheduled at 2025-11-04 09:08:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:08:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:08:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:08:57,774 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:08:58,421 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:08:58,812 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:59,122 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:59,310 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:08:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:08:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:14.351216+00:00 (in 14.999551 seconds) quotatotalworker stdout | 2025-11-04 09:08:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:14 GMT)" (scheduled at 2025-11-04 09:08:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:08:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:08:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:08:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:08:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:08:59,442 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:59,467 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:59,535 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:00,697 [286] [DEBUG] [app] Starting request: urn:request:a935db6a-0710-4f54-8496-c77a868c97e8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:00,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:00,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:00,702 [292] [DEBUG] [app] Starting request: urn:request:a137e8a8-6073-48cc-825b-093a43c93616 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:00,702 [292] [DEBUG] [app] Ending request: urn:request:a137e8a8-6073-48cc-825b-093a43c93616 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a137e8a8-6073-48cc-825b-093a43c93616', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:09:00,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:00,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:00,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:00,706 [285] [DEBUG] [app] Starting request: urn:request:cb791d5a-1b45-4c33-bce1-c93eb2ec84c8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:00,706 [285] [DEBUG] [app] Ending request: urn:request:cb791d5a-1b45-4c33-bce1-c93eb2ec84c8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cb791d5a-1b45-4c33-bce1-c93eb2ec84c8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:00,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:00,707 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:00,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:00,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:00,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:00,715 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:00,715 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:00,725 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:00,728 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:00,731 [286] [DEBUG] [app] Ending request: urn:request:a935db6a-0710-4f54-8496-c77a868c97e8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a935db6a-0710-4f54-8496-c77a868c97e8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:00,731 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:09:00,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:09:00,783 [286] [DEBUG] [app] Starting request: urn:request:97dfb236-ecd6-48f9-b7e6-1f95823eb7e0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:00,784 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:00,785 [299] [DEBUG] [app] Starting request: urn:request:9148bc31-df33-4d49-ab10-33ff19482b17 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:00,785 [299] [DEBUG] [app] Ending request: urn:request:9148bc31-df33-4d49-ab10-33ff19482b17 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9148bc31-df33-4d49-ab10-33ff19482b17', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:00,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:00,786 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:00,786 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:00,789 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:00,789 [288] [DEBUG] [app] Starting request: urn:request:96930c4f-bd77-4d09-bc95-c1437b9a4729 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:00,790 [288] [DEBUG] [app] Ending request: urn:request:96930c4f-bd77-4d09-bc95-c1437b9a4729 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:96930c4f-bd77-4d09-bc95-c1437b9a4729', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:00,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:00,790 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:00,790 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:00,790 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:00,790 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:00,798 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:00,798 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:00,808 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:00,811 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:00,814 [286] [DEBUG] [app] Ending request: urn:request:97dfb236-ecd6-48f9-b7e6-1f95823eb7e0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:97dfb236-ecd6-48f9-b7e6-1f95823eb7e0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:00,814 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.032) gunicorn-web stdout | 2025-11-04 09:09:00,815 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:09:01,062 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:09:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:11.530064+00:00 (in 9.999514 seconds) notificationworker stdout | 2025-11-04 09:09:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:11 GMT)" (scheduled at 2025-11-04 09:09:01.530064+00:00) notificationworker stdout | 2025-11-04 09:09:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 1, 530828), True, datetime.datetime(2025, 11, 4, 9, 9, 1, 530828), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:01,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:09:01,861 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:09:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:09:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:39.924652+00:00 (in 38.001446 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:09:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:01 GMT)" (scheduled at 2025-11-04 09:09:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:09:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:09:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 1, 923461), True, datetime.datetime(2025, 11, 4, 9, 9, 1, 923461), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:09:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:09:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:09:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:01 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:09:02,048 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:09:02,049 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:09:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:09:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:43.432867+00:00 (in 40.909053 seconds) repositorygcworker stdout | 2025-11-04 09:09:02,523 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:02 GMT)" (scheduled at 2025-11-04 09:09:02.523412+00:00) repositorygcworker stdout | 2025-11-04 09:09:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 9, 2, 524040), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:09:02,536 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 2, 524040), True, datetime.datetime(2025, 11, 4, 9, 9, 2, 524040), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:09:02,539 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 2, 524040), True, datetime.datetime(2025, 11, 4, 9, 9, 2, 524040), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 9, 2, 524040), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:09:02,543 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:09:02,543 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:02 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:09:03,342 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:09:03,345 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:09:03,457 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:09:04,196 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:09:04,209 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:09:04,209 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:09:04,219 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:09:04,233 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:09:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:08.437104+00:00 (in 3.000834 seconds) gcworker stdout | 2025-11-04 09:09:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:35 GMT)" (scheduled at 2025-11-04 09:09:05.435809+00:00) gcworker stdout | 2025-11-04 09:09:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037745436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:09:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:09:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:09:05,532 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:09:05,899 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:09:06,522 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:09:06,910 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:17.122849+00:00 (in 9.999558 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:17 GMT)" (scheduled at 2025-11-04 09:09:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 7, 123572), True, datetime.datetime(2025, 11, 4, 9, 9, 7, 123572), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:09:07,757 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:07,774 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:07,800 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:07,822 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:07,820 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:07,841 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:07,872 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:08,002 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:08,021 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:08,248 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:08,248 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:08,264 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:09:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:35.435809+00:00 (in 26.998186 seconds) gcworker stdout | 2025-11-04 09:09:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:38 GMT)" (scheduled at 2025-11-04 09:09:08.437104+00:00) gcworker stdout | 2025-11-04 09:09:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:09:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247288449, None, 1, 0]) gcworker stdout | 2025-11-04 09:09:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:09:09,082 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:09:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:09:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:57.131795+00:00 (in 47.997745 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:09 GMT)" (scheduled at 2025-11-04 09:09:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:09:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:09:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:09:10,853 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:09:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:15.535095+00:00 (in 4.004567 seconds) notificationworker stdout | 2025-11-04 09:09:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:21 GMT)" (scheduled at 2025-11-04 09:09:11.530064+00:00) notificationworker stdout | 2025-11-04 09:09:11,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 11, 531147), True, datetime.datetime(2025, 11, 4, 9, 9, 11, 531147), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:09:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:09:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:12.222828+00:00 (in 59.999576 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:09:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:12 GMT)" (scheduled at 2025-11-04 09:09:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:09:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:09:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:09:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:09:12,295 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:09:14,219 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:09:14,233 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:09:14,233 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:09:14,244 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:09:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:09:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:29.351216+00:00 (in 14.999596 seconds) quotatotalworker stdout | 2025-11-04 09:09:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:29 GMT)" (scheduled at 2025-11-04 09:09:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:09:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:09:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:09:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:09:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:09:14,798 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:09:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:17.479080+00:00 (in 1.998855 seconds) securityworker stdout | 2025-11-04 09:09:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:45 GMT)" (scheduled at 2025-11-04 09:09:15.479769+00:00) securityworker stdout | 2025-11-04 09:09:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:15,481 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:09:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 15, 484078), 1, 2]) securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 15, 484078), 1, 2]) securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:21.530064+00:00 (in 5.994504 seconds) notificationworker stdout | 2025-11-04 09:09:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:15 GMT)" (scheduled at 2025-11-04 09:09:15.535095+00:00) notificationworker stdout | 2025-11-04 09:09:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:09:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:09:15,697 [285] [DEBUG] [app] Starting request: urn:request:ff5bf0d2-03f8-479e-883c-25fbea3278b7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:15,701 [296] [DEBUG] [app] Starting request: urn:request:cfe7c9bb-3c2c-46e8-8535-59765a69838f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:15,702 [296] [DEBUG] [app] Ending request: urn:request:cfe7c9bb-3c2c-46e8-8535-59765a69838f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cfe7c9bb-3c2c-46e8-8535-59765a69838f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:15,702 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:15,706 [288] [DEBUG] [app] Starting request: urn:request:0728ea5e-4ec1-4c99-acef-8692e378b6cc (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:15,706 [288] [DEBUG] [app] Ending request: urn:request:0728ea5e-4ec1-4c99-acef-8692e378b6cc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0728ea5e-4ec1-4c99-acef-8692e378b6cc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:09:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:15,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:15,731 [285] [DEBUG] [app] Ending request: urn:request:ff5bf0d2-03f8-479e-883c-25fbea3278b7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ff5bf0d2-03f8-479e-883c-25fbea3278b7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:15,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:15,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:09:15,783 [285] [DEBUG] [app] Starting request: urn:request:66e17570-e0db-4dcd-856c-ede86482258f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:15,783 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:15,784 [292] [DEBUG] [app] Starting request: urn:request:5e307af8-3e48-425a-8030-e0e42056b944 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:15,785 [292] [DEBUG] [app] Ending request: urn:request:5e307af8-3e48-425a-8030-e0e42056b944 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5e307af8-3e48-425a-8030-e0e42056b944', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:09:15,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:15,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:15,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:15,789 [288] [DEBUG] [app] Starting request: urn:request:ba048699-6178-4cb4-a798-b2e1c04c7461 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:15,789 [288] [DEBUG] [app] Ending request: urn:request:ba048699-6178-4cb4-a798-b2e1c04c7461 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ba048699-6178-4cb4-a798-b2e1c04c7461', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:15,789 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:15,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:15,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:15,813 [285] [DEBUG] [app] Ending request: urn:request:66e17570-e0db-4dcd-856c-ede86482258f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:66e17570-e0db-4dcd-856c-ede86482258f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:15,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.032) proxycacheblobworker stdout | 2025-11-04 09:09:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:27.122849+00:00 (in 9.999621 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:27 GMT)" (scheduled at 2025-11-04 09:09:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 17, 123509), True, datetime.datetime(2025, 11, 4, 9, 9, 17, 123509), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:09:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:45.479769+00:00 (in 28.000233 seconds) securityworker stdout | 2025-11-04 09:09:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:47 GMT)" (scheduled at 2025-11-04 09:09:17.479080+00:00) securityworker stdout | 2025-11-04 09:09:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:09:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:09:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:09:17,741 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:18,399 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:09:19,712 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:09:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:09:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:51.226294+00:00 (in 29.999559 seconds) autopruneworker stdout | 2025-11-04 09:09:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:51 GMT)" (scheduled at 2025-11-04 09:09:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:09:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247301234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:09:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:09:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:09:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:09:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:09:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:32.435813+00:00 (in 11.000955 seconds) namespacegcworker stdout | 2025-11-04 09:09:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:21 GMT)" (scheduled at 2025-11-04 09:09:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:09:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:09:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 21, 435102), True, datetime.datetime(2025, 11, 4, 9, 9, 21, 435102), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:09:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:09:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:09:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:31.530064+00:00 (in 9.999534 seconds) notificationworker stdout | 2025-11-04 09:09:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:31 GMT)" (scheduled at 2025-11-04 09:09:21.530064+00:00) notificationworker stdout | 2025-11-04 09:09:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 21, 530815), True, datetime.datetime(2025, 11, 4, 9, 9, 21, 530815), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:09:21,976 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:09:24,244 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:09:24,258 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:09:24,258 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:09:24,268 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:24,842 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:09:27,067 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:37.122849+00:00 (in 9.999649 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:37 GMT)" (scheduled at 2025-11-04 09:09:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 27, 123452), True, datetime.datetime(2025, 11, 4, 9, 9, 27, 123452), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:09:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:09:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:57.222846+00:00 (in 29.999576 seconds) buildlogsarchiver stdout | 2025-11-04 09:09:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:57 GMT)" (scheduled at 2025-11-04 09:09:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:09:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 9, 27, 223536), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:09:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:09:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:09:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:09:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:09:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:09:27,783 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:09:28,430 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:09:28,821 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:29,133 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:29,322 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:09:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:09:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:44.351216+00:00 (in 14.999583 seconds) quotatotalworker stdout | 2025-11-04 09:09:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:44 GMT)" (scheduled at 2025-11-04 09:09:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:09:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:09:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:09:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:09:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:09:29,456 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:29,482 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:29,549 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:30,696 [285] [DEBUG] [app] Starting request: urn:request:ed172ef3-af47-4720-887d-c3fab1a671ef (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:30,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:30,701 [293] [DEBUG] [app] Starting request: urn:request:a218a971-e6c9-45a4-9215-aff6a37b277d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:30,701 [293] [DEBUG] [app] Ending request: urn:request:a218a971-e6c9-45a4-9215-aff6a37b277d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a218a971-e6c9-45a4-9215-aff6a37b277d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:09:30,701 [293] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:30,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:30,705 [287] [DEBUG] [app] Starting request: urn:request:ce79e9e4-3c17-4f97-abbd-45e13f28005f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:30,705 [287] [DEBUG] [app] Ending request: urn:request:ce79e9e4-3c17-4f97-abbd-45e13f28005f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ce79e9e4-3c17-4f97-abbd-45e13f28005f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:30,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:30,726 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:30,729 [285] [DEBUG] [app] Ending request: urn:request:ed172ef3-af47-4720-887d-c3fab1a671ef (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ed172ef3-af47-4720-887d-c3fab1a671ef', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:09:30,784 [285] [DEBUG] [app] Starting request: urn:request:661baa73-48a6-492b-bf05-19738a0bc2dc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:30,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:30,786 [299] [DEBUG] [app] Starting request: urn:request:19076ea7-1477-4f92-8b12-7c9c25eb49bb (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:30,786 [299] [DEBUG] [app] Ending request: urn:request:19076ea7-1477-4f92-8b12-7c9c25eb49bb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:19076ea7-1477-4f92-8b12-7c9c25eb49bb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:30,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:30,787 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:30,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:30,790 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:30,790 [287] [DEBUG] [app] Starting request: urn:request:74657f08-9c37-4112-88bc-b0dbf9e078b9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:30,791 [287] [DEBUG] [app] Ending request: urn:request:74657f08-9c37-4112-88bc-b0dbf9e078b9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:74657f08-9c37-4112-88bc-b0dbf9e078b9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:30,791 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:30,791 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:30,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:30,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:30,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:30,799 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:30,799 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:30,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:30,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:30,814 [285] [DEBUG] [app] Ending request: urn:request:661baa73-48a6-492b-bf05-19738a0bc2dc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:661baa73-48a6-492b-bf05-19738a0bc2dc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:30,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:30,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:31,071 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:09:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:40.533243+00:00 (in 9.002735 seconds) notificationworker stdout | 2025-11-04 09:09:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:41 GMT)" (scheduled at 2025-11-04 09:09:31.530064+00:00) notificationworker stdout | 2025-11-04 09:09:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 31, 530785), True, datetime.datetime(2025, 11, 4, 9, 9, 31, 530785), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:09:31,871 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:09:32,057 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:09:32,056 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:09:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:09:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:21.434460+00:00 (in 48.998200 seconds) namespacegcworker stdout | 2025-11-04 09:09:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:32 GMT)" (scheduled at 2025-11-04 09:09:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:09:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:09:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:09:33,350 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:09:33,354 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:09:33,466 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:09:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:09:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:48.625192+00:00 (in 15.001852 seconds) exportactionlogsworker stdout | 2025-11-04 09:09:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:33 GMT)" (scheduled at 2025-11-04 09:09:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:09:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:09:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 33, 623612), True, datetime.datetime(2025, 11, 4, 9, 9, 33, 623612), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:09:33,635 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:09:33,635 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:09:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:33 GMT)" executed successfully builder stdout | 2025-11-04 09:09:34,242 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:09:34,268 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:09:34,281 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:09:34,281 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:09:34,291 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:38.437104+00:00 (in 3.000793 seconds) gcworker stdout | 2025-11-04 09:09:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:05 GMT)" (scheduled at 2025-11-04 09:09:35.435809+00:00) gcworker stdout | 2025-11-04 09:09:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037775436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:09:35,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:09:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:35,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:09:35,541 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:09:35,908 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:09:36,532 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:09:36,919 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:40.124445+00:00 (in 3.001215 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:47 GMT)" (scheduled at 2025-11-04 09:09:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 37, 123452), True, datetime.datetime(2025, 11, 4, 9, 9, 37, 123452), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:37,138 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:37,138 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:37,138 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:09:37,771 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:37,786 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:37,812 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:37,835 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:37,845 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:37,855 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:37,882 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:38,013 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:38,034 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:38,258 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:38,259 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:38,274 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:09:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:05.435809+00:00 (in 26.998215 seconds) gcworker stdout | 2025-11-04 09:09:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:08 GMT)" (scheduled at 2025-11-04 09:09:38.437104+00:00) gcworker stdout | 2025-11-04 09:09:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:09:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247318448, None, 1, 0]) gcworker stdout | 2025-11-04 09:09:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:09:39,101 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:09:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:09:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:01.922827+00:00 (in 21.997752 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:09:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:39 GMT)" (scheduled at 2025-11-04 09:09:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:09:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:09:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:09:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:47.122849+00:00 (in 6.997981 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:40 GMT)" (scheduled at 2025-11-04 09:09:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:09:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:41.530064+00:00 (in 0.996327 seconds) notificationworker stdout | 2025-11-04 09:09:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:40 GMT)" (scheduled at 2025-11-04 09:09:40.533243+00:00) notificationworker stdout | 2025-11-04 09:09:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 9, 40, 534016), 'notification/%']) notificationworker stdout | 2025-11-04 09:09:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 40, 534016), True, datetime.datetime(2025, 11, 4, 9, 9, 40, 534016), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:09:40,550 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 40, 534016), True, datetime.datetime(2025, 11, 4, 9, 9, 40, 534016), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 9, 40, 534016), 'notification/%']) notificationworker stdout | 2025-11-04 09:09:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:09:40,861 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:09:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:51.530064+00:00 (in 9.999597 seconds) notificationworker stdout | 2025-11-04 09:09:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:51 GMT)" (scheduled at 2025-11-04 09:09:41.530064+00:00) notificationworker stdout | 2025-11-04 09:09:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 41, 530735), True, datetime.datetime(2025, 11, 4, 9, 9, 41, 530735), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:09:42,303 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:09:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:09:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:57.523915+00:00 (in 14.090661 seconds) repositorygcworker stdout | 2025-11-04 09:09:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:43 GMT)" (scheduled at 2025-11-04 09:09:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:09:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:09:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 43, 433550), True, datetime.datetime(2025, 11, 4, 9, 9, 43, 433550), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:09:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:09:43,445 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:09:43,445 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:43 GMT)" executed successfully builder stdout | 2025-11-04 09:09:44,291 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:09:44,304 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:09:44,304 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:09:44,315 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:09:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:09:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:59.351216+00:00 (in 14.999620 seconds) quotatotalworker stdout | 2025-11-04 09:09:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:59 GMT)" (scheduled at 2025-11-04 09:09:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:09:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:09:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:09:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:09:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:09:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:09:44,807 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:09:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:47.479080+00:00 (in 1.998902 seconds) securityworker stdout | 2025-11-04 09:09:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:15 GMT)" (scheduled at 2025-11-04 09:09:45.479769+00:00) securityworker stdout | 2025-11-04 09:09:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 45, 483190), 1, 2]) securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 45, 483190), 1, 2]) securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:09:45,697 [285] [DEBUG] [app] Starting request: urn:request:6fde0afb-afde-47f8-b96b-9fa354344011 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:45,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:45,702 [292] [DEBUG] [app] Starting request: urn:request:525da8a2-0397-4818-8b73-e36926af41bd (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:45,702 [292] [DEBUG] [app] Ending request: urn:request:525da8a2-0397-4818-8b73-e36926af41bd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:525da8a2-0397-4818-8b73-e36926af41bd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:45,703 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:45,703 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:45,706 [288] [DEBUG] [app] Starting request: urn:request:abe69b00-84d9-4d19-b089-aa6f4be48bce (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:45,707 [288] [DEBUG] [app] Ending request: urn:request:abe69b00-84d9-4d19-b089-aa6f4be48bce (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:abe69b00-84d9-4d19-b089-aa6f4be48bce', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:45,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:45,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:45,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:45,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:45,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:45,729 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:45,732 [285] [DEBUG] [app] Ending request: urn:request:6fde0afb-afde-47f8-b96b-9fa354344011 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6fde0afb-afde-47f8-b96b-9fa354344011', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:45,732 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:45,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:09:45,784 [285] [DEBUG] [app] Starting request: urn:request:164589d3-150c-4140-a74f-9e108047d597 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:09:45,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:09:45,785 [292] [DEBUG] [app] Starting request: urn:request:fc9fe055-a99d-40f1-81f4-96707f2c8e20 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:09:45,786 [292] [DEBUG] [app] Ending request: urn:request:fc9fe055-a99d-40f1-81f4-96707f2c8e20 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fc9fe055-a99d-40f1-81f4-96707f2c8e20', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:45,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:45,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:45,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:45,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:09:45,790 [288] [DEBUG] [app] Starting request: urn:request:9812633e-ebfe-49b9-b79b-066b362d82d9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:09:45,790 [288] [DEBUG] [app] Ending request: urn:request:9812633e-ebfe-49b9-b79b-066b362d82d9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9812633e-ebfe-49b9-b79b-066b362d82d9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:09:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:09:45,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:09:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:45,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:45,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:45,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:45,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:45,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:45,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:45,809 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:45,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:45,816 [285] [DEBUG] [app] Ending request: urn:request:164589d3-150c-4140-a74f-9e108047d597 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:164589d3-150c-4140-a74f-9e108047d597', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:45,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:45,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:09:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:09:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.032) proxycacheblobworker stdout | 2025-11-04 09:09:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:53.124163+00:00 (in 6.000894 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:57 GMT)" (scheduled at 2025-11-04 09:09:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 47, 123615), True, datetime.datetime(2025, 11, 4, 9, 9, 47, 123615), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:09:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:15.479769+00:00 (in 28.000212 seconds) securityworker stdout | 2025-11-04 09:09:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:17 GMT)" (scheduled at 2025-11-04 09:09:47.479080+00:00) securityworker stdout | 2025-11-04 09:09:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:09:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:09:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 47, 483033), 1, 2]) securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:47,505 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 47, 483033), 1, 2]) securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:47,509 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:09:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:09:47,750 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:48,408 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:09:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:09:48,626 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:33.622985+00:00 (in 44.996978 seconds) exportactionlogsworker stdout | 2025-11-04 09:09:48,626 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:48 GMT)" (scheduled at 2025-11-04 09:09:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:09:48,626 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:09:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:09:49,720 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:09:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:09:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:21.226294+00:00 (in 29.999572 seconds) autopruneworker stdout | 2025-11-04 09:09:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:21 GMT)" (scheduled at 2025-11-04 09:09:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:09:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247331235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:09:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:09:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:09:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:01.530064+00:00 (in 9.999621 seconds) notificationworker stdout | 2025-11-04 09:09:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:01 GMT)" (scheduled at 2025-11-04 09:09:51.530064+00:00) notificationworker stdout | 2025-11-04 09:09:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 51, 530764), True, datetime.datetime(2025, 11, 4, 9, 9, 51, 530764), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:51,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:51,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:09:51,984 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:57.122849+00:00 (in 3.998211 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:53 GMT)" (scheduled at 2025-11-04 09:09:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 9, 53, 124870), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:09:53,137 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 53, 124870), True, datetime.datetime(2025, 11, 4, 9, 9, 53, 124870), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:09:53,141 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 53, 124870), True, datetime.datetime(2025, 11, 4, 9, 9, 53, 124870), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 9, 53, 124870), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:09:53,145 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:53,145 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:53 GMT)" executed successfully builder stdout | 2025-11-04 09:09:54,315 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:09:54,329 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:09:54,329 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:09:54,339 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:54,851 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:09:57,076 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:07.122849+00:00 (in 9.999636 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:07 GMT)" (scheduled at 2025-11-04 09:09:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 57, 123467), True, datetime.datetime(2025, 11, 4, 9, 9, 57, 123467), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:09:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:09.133645+00:00 (in 12.001431 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:57 GMT)" (scheduled at 2025-11-04 09:09:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:57,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:57,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:09:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:09:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:09:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:09:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:09:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:27.222846+00:00 (in 29.999586 seconds) buildlogsarchiver stdout | 2025-11-04 09:09:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:27 GMT)" (scheduled at 2025-11-04 09:09:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:09:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 9, 57, 223525), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:09:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:09:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:09:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:09:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:09:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:09:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:09:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:43.432867+00:00 (in 45.908519 seconds) repositorygcworker stdout | 2025-11-04 09:09:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:57 GMT)" (scheduled at 2025-11-04 09:09:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:09:57,525 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:09:57,525 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:09:57,792 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:09:58,439 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:09:58,829 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:59,144 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:59,335 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:09:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:09:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:14.351216+00:00 (in 14.999607 seconds) quotatotalworker stdout | 2025-11-04 09:09:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:14 GMT)" (scheduled at 2025-11-04 09:09:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:09:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:09:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:09:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:09:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:09:59,470 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:59,495 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:59,563 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:00,697 [286] [DEBUG] [app] Starting request: urn:request:a6375d3e-4a27-4436-8bef-0a7e1ffa5e28 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:00,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:00,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:00,701 [292] [DEBUG] [app] Starting request: urn:request:77b90d57-e8c2-4a8f-a3f7-7c5e68bb2f4a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:00,701 [292] [DEBUG] [app] Ending request: urn:request:77b90d57-e8c2-4a8f-a3f7-7c5e68bb2f4a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:77b90d57-e8c2-4a8f-a3f7-7c5e68bb2f4a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:10:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:00,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:00,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:00,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:00,705 [287] [DEBUG] [app] Starting request: urn:request:e5ac79fa-9374-431e-af63-57eba84ec9b0 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:00,706 [287] [DEBUG] [app] Ending request: urn:request:e5ac79fa-9374-431e-af63-57eba84ec9b0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e5ac79fa-9374-431e-af63-57eba84ec9b0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:00,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:00,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:00,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:00,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:00,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:00,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:00,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:00,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:00,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:00,731 [286] [DEBUG] [app] Ending request: urn:request:a6375d3e-4a27-4436-8bef-0a7e1ffa5e28 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a6375d3e-4a27-4436-8bef-0a7e1ffa5e28', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:00,731 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:00,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:10:00,784 [287] [DEBUG] [app] Starting request: urn:request:de52c058-6e01-45bc-adc1-640757663288 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:00,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:00,788 [292] [DEBUG] [app] Starting request: urn:request:550754dc-19f5-4aa6-99fc-ba188392b75e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:00,788 [292] [DEBUG] [app] Ending request: urn:request:550754dc-19f5-4aa6-99fc-ba188392b75e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:550754dc-19f5-4aa6-99fc-ba188392b75e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:10:00,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:00,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:00,790 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:00,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:00,792 [286] [DEBUG] [app] Starting request: urn:request:1cfe778c-9295-41a7-9a05-4d7ef100cba7 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:00,792 [286] [DEBUG] [app] Ending request: urn:request:1cfe778c-9295-41a7-9a05-4d7ef100cba7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1cfe778c-9295-41a7-9a05-4d7ef100cba7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:00,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:00,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:00,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:00,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:00,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:00,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:00,810 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:00,813 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:00,816 [287] [DEBUG] [app] Ending request: urn:request:de52c058-6e01-45bc-adc1-640757663288 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:de52c058-6e01-45bc-adc1-640757663288', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:00,816 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:00,816 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.033) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:01,080 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:10:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:11.530064+00:00 (in 9.999554 seconds) notificationworker stdout | 2025-11-04 09:10:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:11 GMT)" (scheduled at 2025-11-04 09:10:01.530064+00:00) notificationworker stdout | 2025-11-04 09:10:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 1, 530767), True, datetime.datetime(2025, 11, 4, 9, 10, 1, 530767), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:10:01,880 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:10:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:10:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:39.924652+00:00 (in 38.001431 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:10:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:01 GMT)" (scheduled at 2025-11-04 09:10:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:10:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:10:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 1, 923449), True, datetime.datetime(2025, 11, 4, 9, 10, 1, 923449), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:10:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:10:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:10:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:01 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:10:02,066 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:10:02,077 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:10:03,359 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:10:03,362 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:10:03,475 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:10:04,252 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:10:04,339 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:10:04,353 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:10:04,353 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:10:04,364 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:08.437104+00:00 (in 3.000870 seconds) gcworker stdout | 2025-11-04 09:10:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:35 GMT)" (scheduled at 2025-11-04 09:10:05.435809+00:00) gcworker stdout | 2025-11-04 09:10:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037805436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:10:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:10:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:10:05,551 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:10:05,917 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:10:06,541 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:10:06,928 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:17.122849+00:00 (in 9.999613 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:17 GMT)" (scheduled at 2025-11-04 09:10:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 7, 123554), True, datetime.datetime(2025, 11, 4, 9, 10, 7, 123554), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:10:07,783 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:07,796 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:07,823 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:07,857 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:07,873 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:07,848 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:07,892 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:08,025 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:08,046 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:08,269 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:08,269 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:08,285 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:10:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:35.435809+00:00 (in 26.998268 seconds) gcworker stdout | 2025-11-04 09:10:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:38 GMT)" (scheduled at 2025-11-04 09:10:08.437104+00:00) gcworker stdout | 2025-11-04 09:10:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:10:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247348449, None, 1, 0]) gcworker stdout | 2025-11-04 09:10:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:10:09,110 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:10:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:10:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:57.131795+00:00 (in 47.997710 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:09 GMT)" (scheduled at 2025-11-04 09:10:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:10:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:10:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:10:10,870 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:10:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:15.535095+00:00 (in 4.004626 seconds) notificationworker stdout | 2025-11-04 09:10:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:21 GMT)" (scheduled at 2025-11-04 09:10:11.530064+00:00) notificationworker stdout | 2025-11-04 09:10:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 11, 530771), True, datetime.datetime(2025, 11, 4, 9, 10, 11, 530771), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:10:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:10:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:12.222828+00:00 (in 59.999553 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:10:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:12 GMT)" (scheduled at 2025-11-04 09:10:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:10:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:10:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:10:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:10:12,318 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:10:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:10:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:29.351216+00:00 (in 14.999591 seconds) quotatotalworker stdout | 2025-11-04 09:10:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:29 GMT)" (scheduled at 2025-11-04 09:10:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:10:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:10:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:10:14,364 [95] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:10:14,364 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue quotatotalworker stdout | 2025-11-04 09:10:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:29 GMT)" executed successfully builder stdout | 2025-11-04 09:10:14,376 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:10:14,376 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:10:14,386 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:10:14,816 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:10:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:17.479080+00:00 (in 1.998851 seconds) securityworker stdout | 2025-11-04 09:10:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:45 GMT)" (scheduled at 2025-11-04 09:10:15.479769+00:00) securityworker stdout | 2025-11-04 09:10:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:10:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 15, 483586), 1, 2]) securityworker stdout | 2025-11-04 09:10:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 15, 483586), 1, 2]) securityworker stdout | 2025-11-04 09:10:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:21.530064+00:00 (in 5.994537 seconds) notificationworker stdout | 2025-11-04 09:10:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:15 GMT)" (scheduled at 2025-11-04 09:10:15.535095+00:00) notificationworker stdout | 2025-11-04 09:10:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:10:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:10:15,696 [287] [DEBUG] [app] Starting request: urn:request:a6ed87ef-3198-4cea-8f8d-46e5e53452e8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:15,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:15,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:15,700 [292] [DEBUG] [app] Starting request: urn:request:c84876b8-5992-44aa-90ec-4258b7934d74 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:15,701 [292] [DEBUG] [app] Ending request: urn:request:c84876b8-5992-44aa-90ec-4258b7934d74 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c84876b8-5992-44aa-90ec-4258b7934d74', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:15,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:10:15,701 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:15,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:15,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:15,705 [285] [DEBUG] [app] Starting request: urn:request:cecd9bfb-1db4-4f31-8eb1-7490a18a9eb2 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:15,705 [285] [DEBUG] [app] Ending request: urn:request:cecd9bfb-1db4-4f31-8eb1-7490a18a9eb2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cecd9bfb-1db4-4f31-8eb1-7490a18a9eb2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:15,705 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:15,705 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:15,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:15,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:15,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:15,713 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:15,713 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:15,723 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:15,726 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:15,729 [287] [DEBUG] [app] Ending request: urn:request:a6ed87ef-3198-4cea-8f8d-46e5e53452e8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a6ed87ef-3198-4cea-8f8d-46e5e53452e8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:15,729 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:15,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:10:15,783 [285] [DEBUG] [app] Starting request: urn:request:2c084f08-6b2e-44c9-985e-5c66e70590c4 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:15,787 [292] [DEBUG] [app] Starting request: urn:request:c66efe76-0827-4d32-b196-302bac1f2f0b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:15,788 [292] [DEBUG] [app] Ending request: urn:request:c66efe76-0827-4d32-b196-302bac1f2f0b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c66efe76-0827-4d32-b196-302bac1f2f0b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:10:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:15,792 [286] [DEBUG] [app] Starting request: urn:request:7c0ed9b2-8fe1-4fb6-9c96-94872bb9e776 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:15,792 [286] [DEBUG] [app] Ending request: urn:request:7c0ed9b2-8fe1-4fb6-9c96-94872bb9e776 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:7c0ed9b2-8fe1-4fb6-9c96-94872bb9e776', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:15,792 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:15,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:15,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:15,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:15,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:15,816 [285] [DEBUG] [app] Ending request: urn:request:2c084f08-6b2e-44c9-985e-5c66e70590c4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2c084f08-6b2e-44c9-985e-5c66e70590c4', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:10:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:27.122849+00:00 (in 9.999596 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:27 GMT)" (scheduled at 2025-11-04 09:10:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 17, 123530), True, datetime.datetime(2025, 11, 4, 9, 10, 17, 123530), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:10:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:45.479769+00:00 (in 28.000232 seconds) securityworker stdout | 2025-11-04 09:10:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:47 GMT)" (scheduled at 2025-11-04 09:10:17.479080+00:00) securityworker stdout | 2025-11-04 09:10:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:10:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:10:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:10:17,759 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:18,417 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:10:19,729 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:10:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:10:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:51.226294+00:00 (in 29.999556 seconds) autopruneworker stdout | 2025-11-04 09:10:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:51 GMT)" (scheduled at 2025-11-04 09:10:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:10:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247361234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:10:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:10:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:10:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:10:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:10:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:32.435813+00:00 (in 11.000931 seconds) namespacegcworker stdout | 2025-11-04 09:10:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:21 GMT)" (scheduled at 2025-11-04 09:10:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:10:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:10:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 21, 435165), True, datetime.datetime(2025, 11, 4, 9, 10, 21, 435165), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:10:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:10:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:10:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:31.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:10:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:31 GMT)" (scheduled at 2025-11-04 09:10:21.530064+00:00) notificationworker stdout | 2025-11-04 09:10:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 21, 530716), True, datetime.datetime(2025, 11, 4, 9, 10, 21, 530716), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:21,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:10:21,993 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:10:24,386 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:10:24,400 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:10:24,400 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:10:24,411 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:24,859 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:10:27,085 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:37.122849+00:00 (in 9.999638 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:37 GMT)" (scheduled at 2025-11-04 09:10:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 27, 123469), True, datetime.datetime(2025, 11, 4, 9, 10, 27, 123469), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:10:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:10:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:57.222846+00:00 (in 29.999582 seconds) buildlogsarchiver stdout | 2025-11-04 09:10:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:57 GMT)" (scheduled at 2025-11-04 09:10:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:10:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 10, 27, 223536), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:10:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:10:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:10:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:10:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:10:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:10:27,800 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:10:28,448 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:10:28,838 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:29,154 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:29,347 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:10:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:10:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:44.351216+00:00 (in 14.999546 seconds) quotatotalworker stdout | 2025-11-04 09:10:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:44 GMT)" (scheduled at 2025-11-04 09:10:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:10:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:10:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:10:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:10:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:10:29,482 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:29,507 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:29,577 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:30,697 [285] [DEBUG] [app] Starting request: urn:request:c26f5fa5-92e8-48ae-9dff-3cc29fde27a9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:30,701 [299] [DEBUG] [app] Starting request: urn:request:5867131d-c09c-4518-944f-bcd30075a889 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:30,702 [299] [DEBUG] [app] Ending request: urn:request:5867131d-c09c-4518-944f-bcd30075a889 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5867131d-c09c-4518-944f-bcd30075a889', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:10:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:30,706 [287] [DEBUG] [app] Starting request: urn:request:ccaa0907-443c-421e-a1f4-030ac1b20a07 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:30,706 [287] [DEBUG] [app] Ending request: urn:request:ccaa0907-443c-421e-a1f4-030ac1b20a07 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ccaa0907-443c-421e-a1f4-030ac1b20a07', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:10:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:30,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:30,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:30,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:30,731 [285] [DEBUG] [app] Ending request: urn:request:c26f5fa5-92e8-48ae-9dff-3cc29fde27a9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c26f5fa5-92e8-48ae-9dff-3cc29fde27a9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:30,731 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:10:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:10:30,783 [287] [DEBUG] [app] Starting request: urn:request:826e48a3-d7c9-46f5-9055-f711bcca41ec (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:30,784 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:30,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:30,788 [292] [DEBUG] [app] Starting request: urn:request:777f55c4-b81d-416a-8356-4aae21b056fe (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:30,788 [292] [DEBUG] [app] Ending request: urn:request:777f55c4-b81d-416a-8356-4aae21b056fe (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:777f55c4-b81d-416a-8356-4aae21b056fe', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:10:30,788 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:30,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:30,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:30,792 [285] [DEBUG] [app] Starting request: urn:request:ba987427-4d78-45f0-abd7-9dd5837faf4a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:30,792 [285] [DEBUG] [app] Ending request: urn:request:ba987427-4d78-45f0-abd7-9dd5837faf4a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ba987427-4d78-45f0-abd7-9dd5837faf4a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:30,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:30,792 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:30,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:30,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:30,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:30,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:30,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:30,810 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:30,813 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:30,816 [287] [DEBUG] [app] Ending request: urn:request:826e48a3-d7c9-46f5-9055-f711bcca41ec (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:826e48a3-d7c9-46f5-9055-f711bcca41ec', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:30,816 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:30,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:31,089 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:10:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:41.530064+00:00 (in 9.999548 seconds) notificationworker stdout | 2025-11-04 09:10:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:41 GMT)" (scheduled at 2025-11-04 09:10:31.530064+00:00) notificationworker stdout | 2025-11-04 09:10:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 31, 530817), True, datetime.datetime(2025, 11, 4, 9, 10, 31, 530817), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:10:31,889 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:10:32,075 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:10:32,086 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:10:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:10:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:21.434460+00:00 (in 48.998222 seconds) namespacegcworker stdout | 2025-11-04 09:10:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:32 GMT)" (scheduled at 2025-11-04 09:10:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:10:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:10:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:10:33,371 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:10:33,368 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:10:33,484 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:10:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:10:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:48.625192+00:00 (in 15.001280 seconds) exportactionlogsworker stdout | 2025-11-04 09:10:33,624 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:33 GMT)" (scheduled at 2025-11-04 09:10:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:10:33,624 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:10:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 33, 624195), True, datetime.datetime(2025, 11, 4, 9, 10, 33, 624195), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:10:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:10:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:10:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:33 GMT)" executed successfully builder stdout | 2025-11-04 09:10:34,261 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:10:34,411 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:10:34,425 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:10:34,425 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:10:34,436 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:38.437104+00:00 (in 3.000912 seconds) gcworker stdout | 2025-11-04 09:10:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:05 GMT)" (scheduled at 2025-11-04 09:10:35.435809+00:00) gcworker stdout | 2025-11-04 09:10:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037835436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:10:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:10:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:10:35,559 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:10:35,926 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:10:36,551 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:10:36,936 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:40.124445+00:00 (in 3.001174 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:47 GMT)" (scheduled at 2025-11-04 09:10:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 37, 123534), True, datetime.datetime(2025, 11, 4, 9, 10, 37, 123534), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:10:37,807 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:37,797 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:37,855 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:37,870 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:37,892 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:37,892 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:37,907 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:38,037 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:38,059 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:38,294 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:38,280 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:38,280 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:10:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:05.435809+00:00 (in 26.998325 seconds) gcworker stdout | 2025-11-04 09:10:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:08 GMT)" (scheduled at 2025-11-04 09:10:38.437104+00:00) gcworker stdout | 2025-11-04 09:10:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:10:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247378449, None, 1, 0]) gcworker stdout | 2025-11-04 09:10:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:10:39,119 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:10:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:10:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:01.922827+00:00 (in 21.997770 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:10:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:39 GMT)" (scheduled at 2025-11-04 09:10:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:10:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:10:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:10:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:47.122849+00:00 (in 6.997972 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:40 GMT)" (scheduled at 2025-11-04 09:10:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:10:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:10:40,879 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:10:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:51.530064+00:00 (in 9.999551 seconds) notificationworker stdout | 2025-11-04 09:10:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:51 GMT)" (scheduled at 2025-11-04 09:10:41.530064+00:00) notificationworker stdout | 2025-11-04 09:10:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 41, 530877), True, datetime.datetime(2025, 11, 4, 9, 10, 41, 530877), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:10:42,327 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:10:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:10:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:57.523915+00:00 (in 14.090616 seconds) repositorygcworker stdout | 2025-11-04 09:10:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:43 GMT)" (scheduled at 2025-11-04 09:10:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:10:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:10:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 43, 433579), True, datetime.datetime(2025, 11, 4, 9, 10, 43, 433579), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:10:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:10:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:10:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:10:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:10:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:59.351216+00:00 (in 14.999551 seconds) quotatotalworker stdout | 2025-11-04 09:10:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:59 GMT)" (scheduled at 2025-11-04 09:10:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:10:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:10:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:10:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:10:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:10:59 GMT)" executed successfully builder stdout | 2025-11-04 09:10:44,436 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:10:44,450 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:10:44,450 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:10:44,460 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:10:44,824 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:10:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:47.479080+00:00 (in 1.998836 seconds) securityworker stdout | 2025-11-04 09:10:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:15 GMT)" (scheduled at 2025-11-04 09:10:45.479769+00:00) securityworker stdout | 2025-11-04 09:10:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:10:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 45, 483899), 1, 2]) securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 45, 483899), 1, 2]) securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:10:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:10:45,696 [287] [DEBUG] [app] Starting request: urn:request:ce953801-71d7-4823-aaa3-c8119d4ad633 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:45,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:45,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:45,701 [299] [DEBUG] [app] Starting request: urn:request:84ba1140-ec3e-4520-b638-84723e418ed7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:45,701 [299] [DEBUG] [app] Ending request: urn:request:84ba1140-ec3e-4520-b638-84723e418ed7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:84ba1140-ec3e-4520-b638-84723e418ed7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:45,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:10:45,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:45,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:45,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:45,706 [288] [DEBUG] [app] Starting request: urn:request:dd969a36-ba4e-4c13-8be3-5651ee5b3a0e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:45,706 [288] [DEBUG] [app] Ending request: urn:request:dd969a36-ba4e-4c13-8be3-5651ee5b3a0e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dd969a36-ba4e-4c13-8be3-5651ee5b3a0e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:45,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:45,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:45,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:45,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:45,715 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:45,715 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:45,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:45,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:45,730 [287] [DEBUG] [app] Ending request: urn:request:ce953801-71d7-4823-aaa3-c8119d4ad633 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ce953801-71d7-4823-aaa3-c8119d4ad633', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:45,731 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:10:45,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:10:45,783 [285] [DEBUG] [app] Starting request: urn:request:e85e12e8-8c34-4e44-be13-bdbfe6195064 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:10:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:10:45,787 [299] [DEBUG] [app] Starting request: urn:request:73c99736-e93c-44b8-b4d5-35f02415518f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:10:45,788 [299] [DEBUG] [app] Ending request: urn:request:73c99736-e93c-44b8-b4d5-35f02415518f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:73c99736-e93c-44b8-b4d5-35f02415518f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:45,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:10:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:10:45,791 [288] [DEBUG] [app] Starting request: urn:request:b26d4798-47eb-4dc3-bfe5-e402094d52a2 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:10:45,792 [288] [DEBUG] [app] Ending request: urn:request:b26d4798-47eb-4dc3-bfe5-e402094d52a2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b26d4798-47eb-4dc3-bfe5-e402094d52a2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:10:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:10:45,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:10:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:45,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:45,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:45,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:45,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:45,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:45,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:45,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:45,817 [285] [DEBUG] [app] Ending request: urn:request:e85e12e8-8c34-4e44-be13-bdbfe6195064 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e85e12e8-8c34-4e44-be13-bdbfe6195064', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:45,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:45,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:10:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:10:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:10:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:57.122849+00:00 (in 9.999577 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:57 GMT)" (scheduled at 2025-11-04 09:10:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 47, 123577), True, datetime.datetime(2025, 11, 4, 9, 10, 47, 123577), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:10:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:15.479769+00:00 (in 28.000185 seconds) securityworker stdout | 2025-11-04 09:10:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:17 GMT)" (scheduled at 2025-11-04 09:10:47.479080+00:00) securityworker stdout | 2025-11-04 09:10:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:47,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:10:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:10:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 47, 484149), 1, 2]) securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 47, 484149), 1, 2]) securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:10:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:10:47,767 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:48,426 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:10:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:10:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:33.622985+00:00 (in 44.997359 seconds) exportactionlogsworker stdout | 2025-11-04 09:10:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:48 GMT)" (scheduled at 2025-11-04 09:10:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:10:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:10:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:10:49,738 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:10:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:10:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:21.226294+00:00 (in 29.999542 seconds) autopruneworker stdout | 2025-11-04 09:10:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:21 GMT)" (scheduled at 2025-11-04 09:10:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:10:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247391234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:10:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:10:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:10:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:01.530064+00:00 (in 9.999545 seconds) notificationworker stdout | 2025-11-04 09:10:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:01 GMT)" (scheduled at 2025-11-04 09:10:51.530064+00:00) notificationworker stdout | 2025-11-04 09:10:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 51, 530810), True, datetime.datetime(2025, 11, 4, 9, 10, 51, 530810), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:10:52,002 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:10:54,460 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:10:54,473 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:10:54,473 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:10:54,484 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:54,868 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:10:57,094 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:07.122849+00:00 (in 9.999615 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:07 GMT)" (scheduled at 2025-11-04 09:10:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 57, 123548), True, datetime.datetime(2025, 11, 4, 9, 10, 57, 123548), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:10:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:09.133645+00:00 (in 12.001451 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:57 GMT)" (scheduled at 2025-11-04 09:10:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:10:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:10:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:10:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:10:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:10:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:27.222846+00:00 (in 29.999232 seconds) buildlogsarchiver stdout | 2025-11-04 09:10:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:27 GMT)" (scheduled at 2025-11-04 09:10:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:10:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 10, 57, 223847), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:10:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:10:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:10:57,241 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:10:57,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:10:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:10:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:10:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:43.432867+00:00 (in 45.908557 seconds) repositorygcworker stdout | 2025-11-04 09:10:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:57 GMT)" (scheduled at 2025-11-04 09:10:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:10:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:10:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:10:57,821 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:10:58,474 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:10:58,858 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:59,170 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:10:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:10:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:14.351216+00:00 (in 14.999586 seconds) quotatotalworker stdout | 2025-11-04 09:10:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:14 GMT)" (scheduled at 2025-11-04 09:10:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:10:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:10:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:10:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:10:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:10:59,364 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:59,498 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:59,520 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:59,591 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:00,696 [286] [DEBUG] [app] Starting request: urn:request:9017ae59-f366-4c4f-86e2-d15b98a9c9de (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:00,697 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:00,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:00,701 [298] [DEBUG] [app] Starting request: urn:request:0f7ca27a-981f-4903-b923-a967745b5583 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:00,701 [298] [DEBUG] [app] Ending request: urn:request:0f7ca27a-981f-4903-b923-a967745b5583 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0f7ca27a-981f-4903-b923-a967745b5583', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:00,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:11:00,701 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:00,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:00,704 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:00,705 [285] [DEBUG] [app] Starting request: urn:request:264d7afc-d5ec-44a8-8ee1-928504a42fb1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:00,705 [285] [DEBUG] [app] Ending request: urn:request:264d7afc-d5ec-44a8-8ee1-928504a42fb1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:264d7afc-d5ec-44a8-8ee1-928504a42fb1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:00,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:00,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:00,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:00,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:00,713 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:00,713 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:00,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:00,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:00,730 [286] [DEBUG] [app] Ending request: urn:request:9017ae59-f366-4c4f-86e2-d15b98a9c9de (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9017ae59-f366-4c4f-86e2-d15b98a9c9de', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:00,730 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:11:00,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:11:00,784 [288] [DEBUG] [app] Starting request: urn:request:3ace30ba-ca94-4b74-9b7a-c346e69d6156 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:00,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:00,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:00,788 [292] [DEBUG] [app] Starting request: urn:request:5b7fd471-c480-430a-b929-88e5ace440fe (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:00,789 [292] [DEBUG] [app] Ending request: urn:request:5b7fd471-c480-430a-b929-88e5ace440fe (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5b7fd471-c480-430a-b929-88e5ace440fe', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:11:00,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:00,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:00,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:00,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:00,793 [287] [DEBUG] [app] Starting request: urn:request:440a0aee-78c6-4762-ba16-a89b98e0b8e7 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:00,793 [287] [DEBUG] [app] Ending request: urn:request:440a0aee-78c6-4762-ba16-a89b98e0b8e7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:440a0aee-78c6-4762-ba16-a89b98e0b8e7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:00,794 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:00,794 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:00,794 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:00,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:00,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:00,802 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:00,802 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:00,812 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:00,815 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:00,818 [288] [DEBUG] [app] Ending request: urn:request:3ace30ba-ca94-4b74-9b7a-c346e69d6156 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3ace30ba-ca94-4b74-9b7a-c346e69d6156', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:00,818 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:00,819 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:01,100 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:11:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:11.530064+00:00 (in 9.999559 seconds) notificationworker stdout | 2025-11-04 09:11:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:11 GMT)" (scheduled at 2025-11-04 09:11:01.530064+00:00) notificationworker stdout | 2025-11-04 09:11:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 1, 530776), True, datetime.datetime(2025, 11, 4, 9, 11, 1, 530776), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:11:01,898 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:11:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:11:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:39.924652+00:00 (in 38.001431 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:11:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:01 GMT)" (scheduled at 2025-11-04 09:11:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:11:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:11:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 1, 923584), True, datetime.datetime(2025, 11, 4, 9, 11, 1, 923584), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:11:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:11:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:11:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:01 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:11:02,084 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:11:02,095 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:11:03,388 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:11:03,380 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:11:03,493 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:11:04,271 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:11:04,484 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:11:04,498 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:11:04,498 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:11:04,509 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:08.437104+00:00 (in 3.000875 seconds) gcworker stdout | 2025-11-04 09:11:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:35 GMT)" (scheduled at 2025-11-04 09:11:05.435809+00:00) gcworker stdout | 2025-11-04 09:11:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037865436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:11:05,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:11:05,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:11:05,568 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:11:05,935 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:11:06,560 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:11:06,946 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:17.122849+00:00 (in 9.999564 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:17 GMT)" (scheduled at 2025-11-04 09:11:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 7, 123596), True, datetime.datetime(2025, 11, 4, 9, 11, 7, 123596), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:11:07,821 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:07,834 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:07,870 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:07,881 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:07,907 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:07,910 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:07,921 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:08,048 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:08,076 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:08,304 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:08,317 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:08,317 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:11:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:35.435809+00:00 (in 26.998251 seconds) gcworker stdout | 2025-11-04 09:11:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:38 GMT)" (scheduled at 2025-11-04 09:11:08.437104+00:00) gcworker stdout | 2025-11-04 09:11:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:11:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247408449, None, 1, 0]) gcworker stdout | 2025-11-04 09:11:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:11:09,128 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:11:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:11:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:57.131795+00:00 (in 47.997752 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:09 GMT)" (scheduled at 2025-11-04 09:11:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:09,145 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:11:09,145 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:11:09,145 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:11:10,888 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:11:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:15.535095+00:00 (in 4.004579 seconds) notificationworker stdout | 2025-11-04 09:11:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:21 GMT)" (scheduled at 2025-11-04 09:11:11.530064+00:00) notificationworker stdout | 2025-11-04 09:11:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 11, 530798), True, datetime.datetime(2025, 11, 4, 9, 11, 11, 530798), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:11:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:11:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:12.222828+00:00 (in 59.999574 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:11:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:12 GMT)" (scheduled at 2025-11-04 09:11:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:11:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:11:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:11:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:11:12,336 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:11:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:11:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:29.351216+00:00 (in 14.999610 seconds) quotatotalworker stdout | 2025-11-04 09:11:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:29 GMT)" (scheduled at 2025-11-04 09:11:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:11:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:11:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:11:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:11:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:29 GMT)" executed successfully builder stdout | 2025-11-04 09:11:14,509 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:11:14,522 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:11:14,522 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:11:14,532 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:11:14,833 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:11:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:17.479080+00:00 (in 1.998852 seconds) securityworker stdout | 2025-11-04 09:11:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:45 GMT)" (scheduled at 2025-11-04 09:11:15.479769+00:00) securityworker stdout | 2025-11-04 09:11:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:11:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 15, 484014), 1, 2]) securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 15, 484014), 1, 2]) securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:11:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:11:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:11:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:21.530064+00:00 (in 5.994503 seconds) notificationworker stdout | 2025-11-04 09:11:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:15 GMT)" (scheduled at 2025-11-04 09:11:15.535095+00:00) notificationworker stdout | 2025-11-04 09:11:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:11:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:11:15,696 [286] [DEBUG] [app] Starting request: urn:request:f3b37d76-b42e-4890-8439-812e015d4dff (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:15,697 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:15,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:15,701 [297] [DEBUG] [app] Starting request: urn:request:157091db-1090-4cbf-9f66-62af5cc9671f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:15,701 [297] [DEBUG] [app] Ending request: urn:request:157091db-1090-4cbf-9f66-62af5cc9671f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:157091db-1090-4cbf-9f66-62af5cc9671f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:15,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:11:15,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:15,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:15,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:15,705 [285] [DEBUG] [app] Starting request: urn:request:848d47c6-bff7-4dda-b9dc-0bb335ef5802 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:15,706 [285] [DEBUG] [app] Ending request: urn:request:848d47c6-bff7-4dda-b9dc-0bb335ef5802 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:848d47c6-bff7-4dda-b9dc-0bb335ef5802', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:15,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:15,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:15,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:15,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:15,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:15,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:15,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:15,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:15,730 [286] [DEBUG] [app] Ending request: urn:request:f3b37d76-b42e-4890-8439-812e015d4dff (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f3b37d76-b42e-4890-8439-812e015d4dff', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:15,731 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:11:15,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:11:15,783 [285] [DEBUG] [app] Starting request: urn:request:b3e2b84a-2058-4acd-805d-c0b7a051e072 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:15,787 [298] [DEBUG] [app] Starting request: urn:request:982e1b64-5d60-4c35-a1bd-b1741449ec6b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:15,788 [298] [DEBUG] [app] Ending request: urn:request:982e1b64-5d60-4c35-a1bd-b1741449ec6b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:982e1b64-5d60-4c35-a1bd-b1741449ec6b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:15,788 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:11:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:15,792 [287] [DEBUG] [app] Starting request: urn:request:5653ff10-8971-4309-938b-1d2285bfc6d9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:15,792 [287] [DEBUG] [app] Ending request: urn:request:5653ff10-8971-4309-938b-1d2285bfc6d9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5653ff10-8971-4309-938b-1d2285bfc6d9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:15,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:15,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:15,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:15,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:15,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:15,816 [285] [DEBUG] [app] Ending request: urn:request:b3e2b84a-2058-4acd-805d-c0b7a051e072 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b3e2b84a-2058-4acd-805d-c0b7a051e072', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:15,816 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:11:15,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:11:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:27.122849+00:00 (in 9.999598 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:27 GMT)" (scheduled at 2025-11-04 09:11:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 17, 123594), True, datetime.datetime(2025, 11, 4, 9, 11, 17, 123594), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:11:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:45.479769+00:00 (in 28.000194 seconds) securityworker stdout | 2025-11-04 09:11:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:47 GMT)" (scheduled at 2025-11-04 09:11:17.479080+00:00) securityworker stdout | 2025-11-04 09:11:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:11:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:11:17,777 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:18,435 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:11:19,747 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:11:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:11:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:51.226294+00:00 (in 29.999581 seconds) autopruneworker stdout | 2025-11-04 09:11:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:51 GMT)" (scheduled at 2025-11-04 09:11:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:11:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247421234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:11:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:11:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:11:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:11:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:11:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:32.435813+00:00 (in 11.000966 seconds) namespacegcworker stdout | 2025-11-04 09:11:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:21 GMT)" (scheduled at 2025-11-04 09:11:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:11:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:11:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 21, 435150), True, datetime.datetime(2025, 11, 4, 9, 11, 21, 435150), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:11:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:11:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:11:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:31.530064+00:00 (in 9.999603 seconds) notificationworker stdout | 2025-11-04 09:11:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:31 GMT)" (scheduled at 2025-11-04 09:11:21.530064+00:00) notificationworker stdout | 2025-11-04 09:11:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 21, 530738), True, datetime.datetime(2025, 11, 4, 9, 11, 21, 530738), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:31 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:11:21,968 [288] [DEBUG] [app] Starting request: urn:request:73f6eaec-dcbe-4111-be4b-7644aef59240 (/api/v1/user/notifications) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:11:21,969 [288] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:11:21,969 [288] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:11:21,969 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:11:21,981 [288] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:11:21,981 [288] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:11:21,981 [288] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:11:21,982 [288] [DEBUG] [endpoints.api] Checking permission for user whuquay gunicorn-web stdout | 2025-11-04 09:11:21,982 [288] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:11:21,982 [288] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:11:21,982 [288] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:11:21,982 [288] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:11:21,983 [288] [DEBUG] [peewee] ('(SELECT "t1"."id", "t1"."uuid", "t1"."kind_id", "t1"."metadata_json", "t1"."dismissed", "t1"."lookup_path", "t1"."created", "t1"."created" AS "cd", "t1"."target_id" FROM "notification" AS "t1" INNER JOIN "notificationkind" AS "t2" ON ("t1"."kind_id" = "t2"."id") WHERE (("t1"."dismissed" = %s) AND ("t1"."target_id" = %s))) UNION (SELECT "t3"."id", "t3"."uuid", "t3"."kind_id", "t3"."metadata_json", "t3"."dismissed", "t3"."lookup_path", "t3"."created", "t3"."created" AS "cd", "t3"."target_id" FROM "notification" AS "t3" INNER JOIN "notificationkind" AS "t4" ON ("t3"."kind_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t5"."id" = "t3"."target_id") INNER JOIN "team" AS "t6" ON ("t5"."id" = "t6"."organization_id") INNER JOIN "teamrole" AS "t7" ON ("t6"."role_id" = "t7"."id") INNER JOIN "teammember" AS "t8" ON ("t6"."id" = "t8"."team_id") INNER JOIN "user" AS "t9" ON ("t8"."user_id" = "t9"."id") WHERE (("t3"."dismissed" = %s) AND (("t9"."id" = %s) AND ("t7"."name" = %s)))) ORDER BY cd desc LIMIT %s', [False, 1, False, 1, 'admin', 6]) gunicorn-web stdout | 2025-11-04 09:11:21,989 [288] [DEBUG] [app] Ending request: urn:request:73f6eaec-dcbe-4111-be4b-7644aef59240 (/api/v1/user/notifications) {'endpoint': 'api.usernotificationlist', 'request_id': 'urn:request:73f6eaec-dcbe-4111-be4b-7644aef59240', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/user/notifications', 'path': '/api/v1/user/notifications', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:11:21,989 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:11:21 +0000] "GET /api/v1/user/notifications HTTP/2.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.022 956 0.022) gunicorn-web stdout | 2025-11-04 09:11:21,989 [288] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:11:21 +0000] "GET /api/v1/user/notifications HTTP/1.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" autopruneworker stdout | 2025-11-04 09:11:22,010 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:11:24,532 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:11:24,545 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:11:24,545 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:11:24,555 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:24,877 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:11:27,103 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:37.122849+00:00 (in 9.999642 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:37 GMT)" (scheduled at 2025-11-04 09:11:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 27, 123456), True, datetime.datetime(2025, 11, 4, 9, 11, 27, 123456), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:11:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:11:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:57.222846+00:00 (in 29.999597 seconds) buildlogsarchiver stdout | 2025-11-04 09:11:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:57 GMT)" (scheduled at 2025-11-04 09:11:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:11:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 11, 27, 223462), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:11:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:11:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:11:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:11:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:11:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:11:27,830 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:11:28,483 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:11:28,867 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:29,180 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:11:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:11:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:44.351216+00:00 (in 14.999551 seconds) quotatotalworker stdout | 2025-11-04 09:11:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:44 GMT)" (scheduled at 2025-11-04 09:11:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:11:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:11:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:11:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:11:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:11:29,376 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:29,512 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:29,535 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:29,606 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:30,696 [285] [DEBUG] [app] Starting request: urn:request:d2bfe7c7-18f1-45d7-b4f0-6a67b179f0ca (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:30,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:30,701 [292] [DEBUG] [app] Starting request: urn:request:a37b1479-bd37-47c5-8af4-0fbc31890cea (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:30,701 [292] [DEBUG] [app] Ending request: urn:request:a37b1479-bd37-47c5-8af4-0fbc31890cea (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a37b1479-bd37-47c5-8af4-0fbc31890cea', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:11:30,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:30,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:30,705 [287] [DEBUG] [app] Starting request: urn:request:8afb6896-99c3-4d2c-be23-882d2bfe7cca (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:30,705 [287] [DEBUG] [app] Ending request: urn:request:8afb6896-99c3-4d2c-be23-882d2bfe7cca (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8afb6896-99c3-4d2c-be23-882d2bfe7cca', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:30,730 [285] [DEBUG] [app] Ending request: urn:request:d2bfe7c7-18f1-45d7-b4f0-6a67b179f0ca (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d2bfe7c7-18f1-45d7-b4f0-6a67b179f0ca', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:11:30,783 [285] [DEBUG] [app] Starting request: urn:request:91917d07-fe63-4e09-a50d-56e5997c4276 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:30,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:30,785 [292] [DEBUG] [app] Starting request: urn:request:5a6b791c-298a-4000-9dd2-ae6ce2c3d843 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:30,785 [292] [DEBUG] [app] Ending request: urn:request:5a6b791c-298a-4000-9dd2-ae6ce2c3d843 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5a6b791c-298a-4000-9dd2-ae6ce2c3d843', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:11:30,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:30,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:30,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:30,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:30,789 [286] [DEBUG] [app] Starting request: urn:request:02b94536-5c05-4822-8df7-8276e000c5ce (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:30,790 [286] [DEBUG] [app] Ending request: urn:request:02b94536-5c05-4822-8df7-8276e000c5ce (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:02b94536-5c05-4822-8df7-8276e000c5ce', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:30,790 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:30,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:30,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:30,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:30,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:30,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:30,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:30,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:30,813 [285] [DEBUG] [app] Ending request: urn:request:91917d07-fe63-4e09-a50d-56e5997c4276 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:91917d07-fe63-4e09-a50d-56e5997c4276', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:30,813 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.030 47 0.031) gunicorn-web stdout | 2025-11-04 09:11:30,813 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:11:31,109 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:11:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:41.530064+00:00 (in 9.999583 seconds) notificationworker stdout | 2025-11-04 09:11:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:41 GMT)" (scheduled at 2025-11-04 09:11:31.530064+00:00) notificationworker stdout | 2025-11-04 09:11:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 31, 530731), True, datetime.datetime(2025, 11, 4, 9, 11, 31, 530731), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:11:31,907 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:11:32,093 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:11:32,104 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:11:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:11:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:21.434460+00:00 (in 48.998263 seconds) namespacegcworker stdout | 2025-11-04 09:11:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:32 GMT)" (scheduled at 2025-11-04 09:11:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:11:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:11:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:11:33,397 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:11:33,399 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:11:33,502 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:11:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:11:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:48.625192+00:00 (in 15.001845 seconds) exportactionlogsworker stdout | 2025-11-04 09:11:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:33 GMT)" (scheduled at 2025-11-04 09:11:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:11:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:11:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 33, 623634), True, datetime.datetime(2025, 11, 4, 9, 11, 33, 623634), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:11:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:11:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:11:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:33 GMT)" executed successfully builder stdout | 2025-11-04 09:11:34,280 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:11:34,555 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:11:34,568 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:11:34,569 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:11:34,579 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:38.437104+00:00 (in 3.000877 seconds) gcworker stdout | 2025-11-04 09:11:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:05 GMT)" (scheduled at 2025-11-04 09:11:35.435809+00:00) gcworker stdout | 2025-11-04 09:11:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037895436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:11:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:11:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:11:35,577 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:11:35,945 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:11:36,570 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:11:36,956 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:40.124445+00:00 (in 3.001181 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:47 GMT)" (scheduled at 2025-11-04 09:11:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 37, 123605), True, datetime.datetime(2025, 11, 4, 9, 11, 37, 123605), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:37,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:37,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:11:37,836 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:37,845 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:37,884 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:37,893 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:37,918 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:37,922 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:37,934 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:38,063 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:38,087 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:38,314 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:38,327 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:38,328 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:11:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:05.435809+00:00 (in 26.998237 seconds) gcworker stdout | 2025-11-04 09:11:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:08 GMT)" (scheduled at 2025-11-04 09:11:38.437104+00:00) gcworker stdout | 2025-11-04 09:11:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:11:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247438449, None, 1, 0]) gcworker stdout | 2025-11-04 09:11:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:11:39,136 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:11:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:11:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:01.922827+00:00 (in 21.997742 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:11:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:39 GMT)" (scheduled at 2025-11-04 09:11:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:11:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:11:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:11:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:47.122849+00:00 (in 6.998005 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:40 GMT)" (scheduled at 2025-11-04 09:11:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:11:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:11:40,896 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:11:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:51.530064+00:00 (in 9.999542 seconds) notificationworker stdout | 2025-11-04 09:11:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:51 GMT)" (scheduled at 2025-11-04 09:11:41.530064+00:00) notificationworker stdout | 2025-11-04 09:11:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 41, 531019), True, datetime.datetime(2025, 11, 4, 9, 11, 41, 531019), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:11:42,345 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:11:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:11:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:57.523915+00:00 (in 14.090645 seconds) repositorygcworker stdout | 2025-11-04 09:11:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:43 GMT)" (scheduled at 2025-11-04 09:11:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:11:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:11:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 43, 433538), True, datetime.datetime(2025, 11, 4, 9, 11, 43, 433538), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:11:43,447 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:11:43,447 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:11:43,447 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:11:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:11:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:59.351216+00:00 (in 14.999602 seconds) quotatotalworker stdout | 2025-11-04 09:11:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:59 GMT)" (scheduled at 2025-11-04 09:11:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:11:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:11:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:11:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:11:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:11:59 GMT)" executed successfully builder stdout | 2025-11-04 09:11:44,579 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:11:44,592 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:11:44,593 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:11:44,602 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:11:44,853 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:11:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:47.479080+00:00 (in 1.998812 seconds) securityworker stdout | 2025-11-04 09:11:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:15 GMT)" (scheduled at 2025-11-04 09:11:45.479769+00:00) securityworker stdout | 2025-11-04 09:11:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 45, 483767), 1, 2]) securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 45, 483767), 1, 2]) securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:11:45,697 [288] [DEBUG] [app] Starting request: urn:request:ee0a59cc-d251-4f84-9177-2a9fa2c8b343 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:45,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:45,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:45,701 [297] [DEBUG] [app] Starting request: urn:request:d30a16b6-0e9c-428c-8d74-3e3f9269caff (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:45,702 [297] [DEBUG] [app] Ending request: urn:request:d30a16b6-0e9c-428c-8d74-3e3f9269caff (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d30a16b6-0e9c-428c-8d74-3e3f9269caff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:11:45,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:45,706 [285] [DEBUG] [app] Starting request: urn:request:79a45e63-ec13-4d12-8ea9-a0f39bbaf197 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:45,706 [285] [DEBUG] [app] Ending request: urn:request:79a45e63-ec13-4d12-8ea9-a0f39bbaf197 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:79a45e63-ec13-4d12-8ea9-a0f39bbaf197', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:45,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:45,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:45,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:45,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:45,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:45,731 [288] [DEBUG] [app] Ending request: urn:request:ee0a59cc-d251-4f84-9177-2a9fa2c8b343 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ee0a59cc-d251-4f84-9177-2a9fa2c8b343', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:45,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:45,732 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) gunicorn-web stdout | 2025-11-04 09:11:45,783 [286] [DEBUG] [app] Starting request: urn:request:b05a6fe7-efc7-48d3-a4ff-37a6071a3857 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:11:45,784 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:45,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:11:45,788 [299] [DEBUG] [app] Starting request: urn:request:7bc8a464-5dfc-43f1-bc78-0a5f6154d417 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:11:45,788 [299] [DEBUG] [app] Ending request: urn:request:7bc8a464-5dfc-43f1-bc78-0a5f6154d417 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7bc8a464-5dfc-43f1-bc78-0a5f6154d417', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:11:45,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:45,788 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:45,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:45,791 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:11:45,792 [285] [DEBUG] [app] Starting request: urn:request:e41481f4-2b57-4cd2-bc95-cbcecf6f9fcd (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:11:45,792 [285] [DEBUG] [app] Ending request: urn:request:e41481f4-2b57-4cd2-bc95-cbcecf6f9fcd (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e41481f4-2b57-4cd2-bc95-cbcecf6f9fcd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:45,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:11:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:45,792 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:11:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:45,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:45,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:45,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:45,800 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:45,800 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:45,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:45,813 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:45,816 [286] [DEBUG] [app] Ending request: urn:request:b05a6fe7-efc7-48d3-a4ff-37a6071a3857 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b05a6fe7-efc7-48d3-a4ff-37a6071a3857', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:45,816 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:45,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:11:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:11:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:11:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:57.122849+00:00 (in 9.999259 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:57 GMT)" (scheduled at 2025-11-04 09:11:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 47, 123848), True, datetime.datetime(2025, 11, 4, 9, 11, 47, 123848), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:11:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:15.479769+00:00 (in 28.000297 seconds) securityworker stdout | 2025-11-04 09:11:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:17 GMT)" (scheduled at 2025-11-04 09:11:47.479080+00:00) securityworker stdout | 2025-11-04 09:11:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:47,480 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:47,482 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:47,497 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:47,501 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 47, 482748), 1, 2]) securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:11:47,505 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 47, 482748), 1, 2]) securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:47,509 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:11:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:47,509 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:11:47,785 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:48,444 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:11:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:11:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:58.624878+00:00 (in 9.999203 seconds) exportactionlogsworker stdout | 2025-11-04 09:11:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:48 GMT)" (scheduled at 2025-11-04 09:11:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:11:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:11:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:11:49,757 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:11:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:11:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:21.226294+00:00 (in 29.999551 seconds) autopruneworker stdout | 2025-11-04 09:11:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:21 GMT)" (scheduled at 2025-11-04 09:11:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:11:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247451234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:11:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:11:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:11:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:01.530064+00:00 (in 9.999586 seconds) notificationworker stdout | 2025-11-04 09:11:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:01 GMT)" (scheduled at 2025-11-04 09:11:51.530064+00:00) notificationworker stdout | 2025-11-04 09:11:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 51, 530763), True, datetime.datetime(2025, 11, 4, 9, 11, 51, 530763), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:11:52,019 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:11:54,603 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:11:54,616 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:11:54,616 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:11:54,626 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:54,886 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:11:57,112 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:07.122849+00:00 (in 9.999643 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:07 GMT)" (scheduled at 2025-11-04 09:11:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:57,123 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 57, 123433), True, datetime.datetime(2025, 11, 4, 9, 11, 57, 123433), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:11:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:09.133645+00:00 (in 12.001451 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:57 GMT)" (scheduled at 2025-11-04 09:11:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:11:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:11:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:11:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:11:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:11:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:27.222846+00:00 (in 29.999590 seconds) buildlogsarchiver stdout | 2025-11-04 09:11:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:27 GMT)" (scheduled at 2025-11-04 09:11:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:11:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 11, 57, 223461), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:11:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:11:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:11:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:11:57,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:11:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:11:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:11:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:43.432867+00:00 (in 45.908519 seconds) repositorygcworker stdout | 2025-11-04 09:11:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:57 GMT)" (scheduled at 2025-11-04 09:11:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:11:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:11:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:11:57,838 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:11:58,492 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:11:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:11:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:33.622985+00:00 (in 34.997657 seconds) exportactionlogsworker stdout | 2025-11-04 09:11:58,626 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:16:58 GMT)" (scheduled at 2025-11-04 09:11:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:11:58,626 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 11, 58, 626177), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:11:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 11, 58, 626177), True, datetime.datetime(2025, 11, 4, 9, 11, 58, 626177), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:11:58,642 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 11, 58, 626177), True, datetime.datetime(2025, 11, 4, 9, 11, 58, 626177), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 11, 58, 626177), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:11:58,645 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:11:58,645 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:16:58 GMT)" executed successfully teamsyncworker stdout | 2025-11-04 09:11:58,875 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:59,190 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:11:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:11:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:14.351216+00:00 (in 14.999612 seconds) quotatotalworker stdout | 2025-11-04 09:11:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:14 GMT)" (scheduled at 2025-11-04 09:11:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:11:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:11:59,353 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:11:59,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:11:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:11:59,388 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:59,525 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:59,547 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:59,620 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:00,697 [285] [DEBUG] [app] Starting request: urn:request:02b3da1b-2281-4c7f-8fe5-5029e569701d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:00,701 [299] [DEBUG] [app] Starting request: urn:request:ddcbb0b9-d991-453d-85b6-508fe782900f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:00,702 [299] [DEBUG] [app] Ending request: urn:request:ddcbb0b9-d991-453d-85b6-508fe782900f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ddcbb0b9-d991-453d-85b6-508fe782900f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:12:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:00,706 [287] [DEBUG] [app] Starting request: urn:request:efde26f5-59c6-49d2-8c98-f6697839d74f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:00,706 [287] [DEBUG] [app] Ending request: urn:request:efde26f5-59c6-49d2-8c98-f6697839d74f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:efde26f5-59c6-49d2-8c98-f6697839d74f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:00,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:00,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:00,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:00,731 [285] [DEBUG] [app] Ending request: urn:request:02b3da1b-2281-4c7f-8fe5-5029e569701d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:02b3da1b-2281-4c7f-8fe5-5029e569701d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:12:00,784 [285] [DEBUG] [app] Starting request: urn:request:9f757038-f78f-4298-9e1e-ff71dc37cedd (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:00,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:00,785 [292] [DEBUG] [app] Starting request: urn:request:04ac8245-042d-4924-aa4e-f3d65452a3ee (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:00,785 [292] [DEBUG] [app] Ending request: urn:request:04ac8245-042d-4924-aa4e-f3d65452a3ee (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:04ac8245-042d-4924-aa4e-f3d65452a3ee', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:00,790 [288] [DEBUG] [app] Starting request: urn:request:36a46129-7055-4310-b483-136efa6f7c03 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:00,790 [288] [DEBUG] [app] Ending request: urn:request:36a46129-7055-4310-b483-136efa6f7c03 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:36a46129-7055-4310-b483-136efa6f7c03', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:00,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:00,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:00,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:00,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:00,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:00,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:00,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:00,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:00,814 [285] [DEBUG] [app] Ending request: urn:request:9f757038-f78f-4298-9e1e-ff71dc37cedd (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9f757038-f78f-4298-9e1e-ff71dc37cedd', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:00,814 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.032) gunicorn-web stdout | 2025-11-04 09:12:00,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:12:01,118 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:12:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:11.530064+00:00 (in 9.999549 seconds) notificationworker stdout | 2025-11-04 09:12:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:11 GMT)" (scheduled at 2025-11-04 09:12:01.530064+00:00) notificationworker stdout | 2025-11-04 09:12:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 1, 530815), True, datetime.datetime(2025, 11, 4, 9, 12, 1, 530815), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:01,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:11 GMT)" executed successfully gcworker stdout | 2025-11-04 09:12:01,916 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:12:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:12:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:21.924304+00:00 (in 20.001068 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:12:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:01 GMT)" (scheduled at 2025-11-04 09:12:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:12:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:12:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 1, 923538), True, datetime.datetime(2025, 11, 4, 9, 12, 1, 923538), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:12:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:12:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:12:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:01 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:12:02,101 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:12:02,113 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:12:03,406 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:12:03,408 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:12:03,511 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:12:04,289 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:12:04,626 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:12:04,640 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:12:04,640 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:12:04,650 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:08.437104+00:00 (in 3.000912 seconds) gcworker stdout | 2025-11-04 09:12:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:35 GMT)" (scheduled at 2025-11-04 09:12:05.435809+00:00) gcworker stdout | 2025-11-04 09:12:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037925436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:12:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:12:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:12:05,586 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:12:05,953 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:12:06,579 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:12:06,965 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:17.122849+00:00 (in 9.999615 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:17 GMT)" (scheduled at 2025-11-04 09:12:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 7, 123464), True, datetime.datetime(2025, 11, 4, 9, 12, 7, 123464), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:12:07,850 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:07,856 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:07,898 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:07,904 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:07,932 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:07,933 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:07,946 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:08,078 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:08,102 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:08,324 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:08,337 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:08,338 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:12:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:35.435809+00:00 (in 26.998336 seconds) gcworker stdout | 2025-11-04 09:12:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:38 GMT)" (scheduled at 2025-11-04 09:12:08.437104+00:00) gcworker stdout | 2025-11-04 09:12:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:12:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247468449, None, 1, 0]) gcworker stdout | 2025-11-04 09:12:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:38 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:12:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:12:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:57.131795+00:00 (in 47.997689 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:09 GMT)" (scheduled at 2025-11-04 09:12:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:12:09,145 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:12:09,147 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:12:09,147 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:12:09,147 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:09 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:12:10,905 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:12:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:15.535095+00:00 (in 4.004633 seconds) notificationworker stdout | 2025-11-04 09:12:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:21 GMT)" (scheduled at 2025-11-04 09:12:11.530064+00:00) notificationworker stdout | 2025-11-04 09:12:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 11, 531075), True, datetime.datetime(2025, 11, 4, 9, 12, 11, 531075), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:12:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:12:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:12.222828+00:00 (in 59.999571 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:12:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:12 GMT)" (scheduled at 2025-11-04 09:12:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:12:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:12:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:12:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:12:12,354 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:12:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:12:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:29.351216+00:00 (in 14.999557 seconds) quotatotalworker stdout | 2025-11-04 09:12:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:29 GMT)" (scheduled at 2025-11-04 09:12:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:12:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:12:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:12:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:12:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:29 GMT)" executed successfully builder stdout | 2025-11-04 09:12:14,650 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:12:14,664 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:12:14,664 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:12:14,674 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:12:14,862 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:12:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:17.479080+00:00 (in 1.998842 seconds) securityworker stdout | 2025-11-04 09:12:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:45 GMT)" (scheduled at 2025-11-04 09:12:15.479769+00:00) securityworker stdout | 2025-11-04 09:12:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:12:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 15, 483992), 1, 2]) securityworker stdout | 2025-11-04 09:12:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 15, 483992), 1, 2]) securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:21.530064+00:00 (in 5.994544 seconds) notificationworker stdout | 2025-11-04 09:12:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:15 GMT)" (scheduled at 2025-11-04 09:12:15.535095+00:00) notificationworker stdout | 2025-11-04 09:12:15,536 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:12:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:12:15,697 [285] [DEBUG] [app] Starting request: urn:request:b449527a-ffa5-4db0-9b31-7b717b56429f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:15,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:15,702 [299] [DEBUG] [app] Starting request: urn:request:c8afac63-4b2f-4e78-a085-ad4dc9238588 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:15,702 [299] [DEBUG] [app] Ending request: urn:request:c8afac63-4b2f-4e78-a085-ad4dc9238588 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c8afac63-4b2f-4e78-a085-ad4dc9238588', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:12:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:15,706 [288] [DEBUG] [app] Starting request: urn:request:204941e5-5034-464f-ac67-f753bdeaf840 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:15,706 [288] [DEBUG] [app] Ending request: urn:request:204941e5-5034-464f-ac67-f753bdeaf840 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:204941e5-5034-464f-ac67-f753bdeaf840', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:15,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:15,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:15,717 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:15,717 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:15,727 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:15,730 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:15,733 [285] [DEBUG] [app] Ending request: urn:request:b449527a-ffa5-4db0-9b31-7b717b56429f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b449527a-ffa5-4db0-9b31-7b717b56429f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:15,733 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:15,733 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.037 47 0.037) gunicorn-web stdout | 2025-11-04 09:12:15,783 [285] [DEBUG] [app] Starting request: urn:request:1a52203d-6efb-4f7c-a3c0-f0150d67cbd6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:15,784 [299] [DEBUG] [app] Starting request: urn:request:ab3fa00e-48cf-4e75-a387-07dce4f8a72b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:15,785 [299] [DEBUG] [app] Ending request: urn:request:ab3fa00e-48cf-4e75-a387-07dce4f8a72b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ab3fa00e-48cf-4e75-a387-07dce4f8a72b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:15,785 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:15,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:15,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:15,789 [288] [DEBUG] [app] Starting request: urn:request:6de483de-4431-45d6-b6ff-2b700553db58 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:15,789 [288] [DEBUG] [app] Ending request: urn:request:6de483de-4431-45d6-b6ff-2b700553db58 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6de483de-4431-45d6-b6ff-2b700553db58', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:15,789 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:15,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:15,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:15,813 [285] [DEBUG] [app] Ending request: urn:request:1a52203d-6efb-4f7c-a3c0-f0150d67cbd6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1a52203d-6efb-4f7c-a3c0-f0150d67cbd6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:15,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:12:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:27.122849+00:00 (in 9.999592 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:27 GMT)" (scheduled at 2025-11-04 09:12:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 17, 123550), True, datetime.datetime(2025, 11, 4, 9, 12, 17, 123550), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:12:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:45.479769+00:00 (in 28.000264 seconds) securityworker stdout | 2025-11-04 09:12:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:47 GMT)" (scheduled at 2025-11-04 09:12:17.479080+00:00) securityworker stdout | 2025-11-04 09:12:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:17,480 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:17,482 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:12:17,493 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:12:17,493 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:12:17,794 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:18,453 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:12:19,766 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:12:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:12:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:51.226294+00:00 (in 29.999568 seconds) autopruneworker stdout | 2025-11-04 09:12:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:51 GMT)" (scheduled at 2025-11-04 09:12:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:12:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247481234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:12:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:12:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:12:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:12:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:12:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:32.435813+00:00 (in 11.000944 seconds) namespacegcworker stdout | 2025-11-04 09:12:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:21 GMT)" (scheduled at 2025-11-04 09:12:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:12:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:12:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 21, 435209), True, datetime.datetime(2025, 11, 4, 9, 12, 21, 435209), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:12:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:12:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:12:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:31.530064+00:00 (in 9.999625 seconds) notificationworker stdout | 2025-11-04 09:12:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:31 GMT)" (scheduled at 2025-11-04 09:12:21.530064+00:00) notificationworker stdout | 2025-11-04 09:12:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 21, 530710), True, datetime.datetime(2025, 11, 4, 9, 12, 21, 530710), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:21,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:31 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:12:21,925 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:12:21,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:39.924652+00:00 (in 17.999328 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:12:21,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:21 GMT)" (scheduled at 2025-11-04 09:12:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:12:21,926 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 12, 21, 925618), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:12:21,937 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 21, 925618), True, datetime.datetime(2025, 11, 4, 9, 12, 21, 925618), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:12:21,941 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 21, 925618), True, datetime.datetime(2025, 11, 4, 9, 12, 21, 925618), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 12, 21, 925618), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:12:21,944 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:12:21,944 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:21 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:12:22,027 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:12:24,675 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:12:24,688 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:12:24,688 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:12:24,698 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:24,895 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:37.122849+00:00 (in 9.999642 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:37 GMT)" (scheduled at 2025-11-04 09:12:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. quotaregistrysizeworker stdout | 2025-11-04 09:12:27,122 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 27, 123455), True, datetime.datetime(2025, 11, 4, 9, 12, 27, 123455), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:12:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:12:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:57.222846+00:00 (in 29.999597 seconds) buildlogsarchiver stdout | 2025-11-04 09:12:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:57 GMT)" (scheduled at 2025-11-04 09:12:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:12:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 12, 27, 223467), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:12:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:12:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:12:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:12:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:12:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:12:27,847 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:12:28,501 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:12:28,884 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:29,201 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:12:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:12:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:44.351216+00:00 (in 14.999571 seconds) quotatotalworker stdout | 2025-11-04 09:12:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:44 GMT)" (scheduled at 2025-11-04 09:12:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:12:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:12:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:12:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:12:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:12:29,400 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:29,539 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:29,561 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:29,634 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:30,697 [285] [DEBUG] [app] Starting request: urn:request:827c9ecf-31f2-43cd-b293-ba4457cae210 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:30,701 [292] [DEBUG] [app] Starting request: urn:request:5583fb9d-6de6-4cda-bf1b-be014a50892e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:30,701 [292] [DEBUG] [app] Ending request: urn:request:5583fb9d-6de6-4cda-bf1b-be014a50892e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5583fb9d-6de6-4cda-bf1b-be014a50892e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:30,705 [288] [DEBUG] [app] Starting request: urn:request:9d2b5022-7d43-436c-a296-ff09333e1c20 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:30,706 [288] [DEBUG] [app] Ending request: urn:request:9d2b5022-7d43-436c-a296-ff09333e1c20 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9d2b5022-7d43-436c-a296-ff09333e1c20', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:30,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:30,730 [285] [DEBUG] [app] Ending request: urn:request:827c9ecf-31f2-43cd-b293-ba4457cae210 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:827c9ecf-31f2-43cd-b293-ba4457cae210', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:12:30,784 [287] [DEBUG] [app] Starting request: urn:request:84f86db2-6a96-402f-b637-8d5e12a73268 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:30,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:30,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:30,788 [292] [DEBUG] [app] Starting request: urn:request:fbe090b2-1a8a-4f53-8886-fabbd67e5d67 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:30,788 [292] [DEBUG] [app] Ending request: urn:request:fbe090b2-1a8a-4f53-8886-fabbd67e5d67 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fbe090b2-1a8a-4f53-8886-fabbd67e5d67', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:12:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:30,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:30,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:30,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:30,792 [285] [DEBUG] [app] Starting request: urn:request:f3de1be2-84b4-45bd-806f-da28ab416383 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:30,792 [285] [DEBUG] [app] Ending request: urn:request:f3de1be2-84b4-45bd-806f-da28ab416383 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f3de1be2-84b4-45bd-806f-da28ab416383', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:30,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:30,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.000 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:30,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:30,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:30,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:30,803 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:30,803 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:30,812 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:30,816 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:30,819 [287] [DEBUG] [app] Ending request: urn:request:84f86db2-6a96-402f-b637-8d5e12a73268 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:84f86db2-6a96-402f-b637-8d5e12a73268', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:30,819 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:30,819 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:31,127 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:12:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:41.530064+00:00 (in 9.999549 seconds) notificationworker stdout | 2025-11-04 09:12:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:41 GMT)" (scheduled at 2025-11-04 09:12:31.530064+00:00) notificationworker stdout | 2025-11-04 09:12:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 31, 530830), True, datetime.datetime(2025, 11, 4, 9, 12, 31, 530830), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:12:31,924 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:12:32,122 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:12:32,122 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:12:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:12:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:08.435432+00:00 (in 35.999210 seconds) namespacegcworker stdout | 2025-11-04 09:12:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:32 GMT)" (scheduled at 2025-11-04 09:12:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:12:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:12:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:12:33,416 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:12:33,417 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:12:33,519 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:12:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:12:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:48.625192+00:00 (in 15.001853 seconds) exportactionlogsworker stdout | 2025-11-04 09:12:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:33 GMT)" (scheduled at 2025-11-04 09:12:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:12:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:12:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 33, 623666), True, datetime.datetime(2025, 11, 4, 9, 12, 33, 623666), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:12:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:12:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:12:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:33 GMT)" executed successfully builder stdout | 2025-11-04 09:12:34,299 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:12:34,698 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:12:34,712 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:12:34,712 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:12:34,722 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:38.437104+00:00 (in 3.000810 seconds) gcworker stdout | 2025-11-04 09:12:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:05 GMT)" (scheduled at 2025-11-04 09:12:35.435809+00:00) gcworker stdout | 2025-11-04 09:12:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037955436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:12:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:12:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:12:35,595 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:12:35,962 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:12:36,590 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:12:36,974 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:40.124445+00:00 (in 3.001201 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:47 GMT)" (scheduled at 2025-11-04 09:12:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 37, 123512), True, datetime.datetime(2025, 11, 4, 9, 12, 37, 123512), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:12:37,862 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:37,866 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:37,910 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:37,914 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:37,943 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:37,943 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:37,981 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:38,100 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:38,117 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:38,334 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:38,346 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:38,349 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:12:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:05.435809+00:00 (in 26.998337 seconds) gcworker stdout | 2025-11-04 09:12:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:08 GMT)" (scheduled at 2025-11-04 09:12:38.437104+00:00) gcworker stdout | 2025-11-04 09:12:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:12:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247498449, None, 1, 0]) gcworker stdout | 2025-11-04 09:12:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:12:39,154 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:12:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:12:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:01.922827+00:00 (in 21.997712 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:12:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:39 GMT)" (scheduled at 2025-11-04 09:12:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:12:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:12:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:12:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:47.122849+00:00 (in 6.998005 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:40 GMT)" (scheduled at 2025-11-04 09:12:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:12:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:12:40,914 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:12:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:51.530064+00:00 (in 9.999605 seconds) notificationworker stdout | 2025-11-04 09:12:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:51 GMT)" (scheduled at 2025-11-04 09:12:41.530064+00:00) notificationworker stdout | 2025-11-04 09:12:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 41, 530770), True, datetime.datetime(2025, 11, 4, 9, 12, 41, 530770), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:12:42,362 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:12:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:12:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:57.523915+00:00 (in 14.090636 seconds) repositorygcworker stdout | 2025-11-04 09:12:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:43 GMT)" (scheduled at 2025-11-04 09:12:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:12:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:12:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 43, 433605), True, datetime.datetime(2025, 11, 4, 9, 12, 43, 433605), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:12:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:12:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:12:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:12:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:12:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:59.351216+00:00 (in 14.999622 seconds) quotatotalworker stdout | 2025-11-04 09:12:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:59 GMT)" (scheduled at 2025-11-04 09:12:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:12:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:12:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:12:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:12:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:12:59 GMT)" executed successfully builder stdout | 2025-11-04 09:12:44,723 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:12:44,736 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:12:44,736 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:12:44,746 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:12:44,870 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:12:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:47.479080+00:00 (in 1.998848 seconds) securityworker stdout | 2025-11-04 09:12:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:15 GMT)" (scheduled at 2025-11-04 09:12:45.479769+00:00) securityworker stdout | 2025-11-04 09:12:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:12:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:12:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 45, 483714), 1, 2]) securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:12:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 45, 483714), 1, 2]) securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:12:45,697 [286] [DEBUG] [app] Starting request: urn:request:08d13df2-99b4-4ae5-a55a-dc338291b377 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:45,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:45,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:45,701 [292] [DEBUG] [app] Starting request: urn:request:55ff0129-9c31-4e38-a7ca-bd927b7a77d6 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:45,701 [292] [DEBUG] [app] Ending request: urn:request:55ff0129-9c31-4e38-a7ca-bd927b7a77d6 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:55ff0129-9c31-4e38-a7ca-bd927b7a77d6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:45,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:45,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:45,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:45,706 [285] [DEBUG] [app] Starting request: urn:request:33623d46-0ab9-4055-95a9-de21fc4e5cea (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:45,706 [285] [DEBUG] [app] Ending request: urn:request:33623d46-0ab9-4055-95a9-de21fc4e5cea (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:33623d46-0ab9-4055-95a9-de21fc4e5cea', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:45,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:45,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:45,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:45,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:45,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:45,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:45,725 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:45,729 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:45,732 [286] [DEBUG] [app] Ending request: urn:request:08d13df2-99b4-4ae5-a55a-dc338291b377 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:08d13df2-99b4-4ae5-a55a-dc338291b377', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:45,732 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:45,732 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.037) gunicorn-web stdout | 2025-11-04 09:12:45,784 [285] [DEBUG] [app] Starting request: urn:request:2072f5ae-b760-4ff5-94ec-71a26a225cb9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:12:45,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:12:45,788 [292] [DEBUG] [app] Starting request: urn:request:557fa7d7-6dc4-45b1-872d-d14ade536563 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:12:45,788 [292] [DEBUG] [app] Ending request: urn:request:557fa7d7-6dc4-45b1-872d-d14ade536563 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:557fa7d7-6dc4-45b1-872d-d14ade536563', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:12:45,792 [288] [DEBUG] [app] Starting request: urn:request:64470773-dc3f-49c3-825b-89596a2bef28 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:12:45,793 [288] [DEBUG] [app] Ending request: urn:request:64470773-dc3f-49c3-825b-89596a2bef28 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:64470773-dc3f-49c3-825b-89596a2bef28', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:45,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:12:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:12:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:45,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:45,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:45,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:45,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:45,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:45,817 [285] [DEBUG] [app] Ending request: urn:request:2072f5ae-b760-4ff5-94ec-71a26a225cb9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2072f5ae-b760-4ff5-94ec-71a26a225cb9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:45,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:45,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:12:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:12:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:12:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:57.122849+00:00 (in 9.999575 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:57 GMT)" (scheduled at 2025-11-04 09:12:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 47, 123540), True, datetime.datetime(2025, 11, 4, 9, 12, 47, 123540), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:12:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:15.479769+00:00 (in 28.000231 seconds) securityworker stdout | 2025-11-04 09:12:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:17 GMT)" (scheduled at 2025-11-04 09:12:47.479080+00:00) securityworker stdout | 2025-11-04 09:12:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:12:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:12:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:12:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 47, 482868), 1, 2]) securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 47, 482868), 1, 2]) securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:12:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:12:47,803 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:48,462 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:12:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:12:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:33.622985+00:00 (in 44.997406 seconds) exportactionlogsworker stdout | 2025-11-04 09:12:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:48 GMT)" (scheduled at 2025-11-04 09:12:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:12:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:12:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:12:49,776 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:12:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:12:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:21.226294+00:00 (in 29.999561 seconds) autopruneworker stdout | 2025-11-04 09:12:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:21 GMT)" (scheduled at 2025-11-04 09:12:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:12:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247511234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:12:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:12:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:12:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:01.530064+00:00 (in 9.999549 seconds) notificationworker stdout | 2025-11-04 09:12:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:01 GMT)" (scheduled at 2025-11-04 09:12:51.530064+00:00) notificationworker stdout | 2025-11-04 09:12:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 51, 530781), True, datetime.datetime(2025, 11, 4, 9, 12, 51, 530781), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:12:52,047 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:12:54,746 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:12:54,760 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:12:54,760 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:12:54,770 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:54,904 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:07.122849+00:00 (in 9.999618 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:07 GMT)" (scheduled at 2025-11-04 09:12:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 57, 123513), True, datetime.datetime(2025, 11, 4, 9, 12, 57, 123513), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:12:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:09.133645+00:00 (in 12.001455 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:57 GMT)" (scheduled at 2025-11-04 09:12:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:12:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:12:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:12:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:12:57,131 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:12:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:12:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:27.222846+00:00 (in 29.999592 seconds) buildlogsarchiver stdout | 2025-11-04 09:12:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:27 GMT)" (scheduled at 2025-11-04 09:12:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:12:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 12, 57, 223482), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:12:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:12:57,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:12:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:12:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:12:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:12:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:12:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:43.432867+00:00 (in 45.908498 seconds) repositorygcworker stdout | 2025-11-04 09:12:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:57 GMT)" (scheduled at 2025-11-04 09:12:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:12:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:12:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:12:57,856 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:12:58,510 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:12:58,893 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:59,210 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:12:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:12:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:14.351216+00:00 (in 14.999615 seconds) quotatotalworker stdout | 2025-11-04 09:12:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:14 GMT)" (scheduled at 2025-11-04 09:12:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:12:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:12:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:12:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:12:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:12:59,411 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:59,553 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:59,575 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:59,647 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:00,697 [288] [DEBUG] [app] Starting request: urn:request:22d99de6-339c-412c-aa4f-30962c1d0245 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:00,701 [299] [DEBUG] [app] Starting request: urn:request:6417b03f-5f10-4738-8ee4-dcedfcaa2619 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:00,702 [299] [DEBUG] [app] Ending request: urn:request:6417b03f-5f10-4738-8ee4-dcedfcaa2619 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6417b03f-5f10-4738-8ee4-dcedfcaa2619', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:00,706 [285] [DEBUG] [app] Starting request: urn:request:e8cd1757-e644-4747-ad7a-f3f2bf0c6f93 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:00,706 [285] [DEBUG] [app] Ending request: urn:request:e8cd1757-e644-4747-ad7a-f3f2bf0c6f93 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e8cd1757-e644-4747-ad7a-f3f2bf0c6f93', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:00,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:00,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:00,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:00,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:00,730 [288] [DEBUG] [app] Ending request: urn:request:22d99de6-339c-412c-aa4f-30962c1d0245 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:22d99de6-339c-412c-aa4f-30962c1d0245', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:00,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:13:00,784 [285] [DEBUG] [app] Starting request: urn:request:289aa3f3-102a-4076-8413-ccb9161d61ec (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:00,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:00,788 [299] [DEBUG] [app] Starting request: urn:request:6b97edc2-23cf-4fa7-a978-b1f9f24ee15c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:00,788 [299] [DEBUG] [app] Ending request: urn:request:6b97edc2-23cf-4fa7-a978-b1f9f24ee15c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6b97edc2-23cf-4fa7-a978-b1f9f24ee15c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:13:00,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:00,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:00,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:00,792 [288] [DEBUG] [app] Starting request: urn:request:9f5d51ca-6577-442d-9730-271f417613ad (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:00,792 [288] [DEBUG] [app] Ending request: urn:request:9f5d51ca-6577-442d-9730-271f417613ad (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9f5d51ca-6577-442d-9730-271f417613ad', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:00,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:00,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:00,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:00,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:00,817 [285] [DEBUG] [app] Ending request: urn:request:289aa3f3-102a-4076-8413-ccb9161d61ec (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:289aa3f3-102a-4076-8413-ccb9161d61ec', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:00,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:01,136 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:13:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:11.530064+00:00 (in 9.999613 seconds) notificationworker stdout | 2025-11-04 09:13:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:11 GMT)" (scheduled at 2025-11-04 09:13:01.530064+00:00) notificationworker stdout | 2025-11-04 09:13:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 1, 530737), True, datetime.datetime(2025, 11, 4, 9, 13, 1, 530737), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:13:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:13:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:39.924652+00:00 (in 38.001387 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:13:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:01 GMT)" (scheduled at 2025-11-04 09:13:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:13:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:13:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 1, 923697), True, datetime.datetime(2025, 11, 4, 9, 13, 1, 923697), 0, 'secscanv4/%', 50, 1, 0]) gcworker stdout | 2025-11-04 09:13:01,933 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:13:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:13:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:13:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:01 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:13:02,131 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:13:02,131 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:13:03,425 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:13:03,426 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:13:03,528 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:13:04,308 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:13:04,770 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:13:04,783 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:13:04,783 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:13:04,793 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:08.437104+00:00 (in 3.000852 seconds) gcworker stdout | 2025-11-04 09:13:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:35 GMT)" (scheduled at 2025-11-04 09:13:05.435809+00:00) gcworker stdout | 2025-11-04 09:13:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037985436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:13:05,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:13:05,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:13:05,603 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:13:05,972 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:13:06,599 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:13:06,994 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:17.122849+00:00 (in 9.999608 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:17 GMT)" (scheduled at 2025-11-04 09:13:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 7, 123518), True, datetime.datetime(2025, 11, 4, 9, 13, 7, 123518), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:13:07,875 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:07,877 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:07,922 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:07,929 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:07,957 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:07,953 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:07,993 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:08,115 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:08,132 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:08,357 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:08,344 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:08,362 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:13:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:13:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:21.434460+00:00 (in 12.998647 seconds) namespacegcworker stdout | 2025-11-04 09:13:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:18:08 GMT)" (scheduled at 2025-11-04 09:13:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:13:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 13, 8, 436056), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:13:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:35.435809+00:00 (in 26.998351 seconds) gcworker stdout | 2025-11-04 09:13:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:38 GMT)" (scheduled at 2025-11-04 09:13:08.437104+00:00) gcworker stdout | 2025-11-04 09:13:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) namespacegcworker stdout | 2025-11-04 09:13:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 13, 8, 436056), True, datetime.datetime(2025, 11, 4, 9, 13, 8, 436056), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:13:08,448 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247528448, None, 1, 0]) gcworker stdout | 2025-11-04 09:13:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:08,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:13:08,460 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 13, 8, 436056), True, datetime.datetime(2025, 11, 4, 9, 13, 8, 436056), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 13, 8, 436056), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:13:08,463 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:13:08,463 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:18:08 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:13:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:13:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:57.131795+00:00 (in 47.997704 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:09 GMT)" (scheduled at 2025-11-04 09:13:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:13:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:13:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:13:09,163 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:13:10,933 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:13:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:15.535095+00:00 (in 4.004555 seconds) notificationworker stdout | 2025-11-04 09:13:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:21 GMT)" (scheduled at 2025-11-04 09:13:11.530064+00:00) notificationworker stdout | 2025-11-04 09:13:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 11, 530817), True, datetime.datetime(2025, 11, 4, 9, 13, 11, 530817), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:11,545 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:11,545 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:11,545 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:13:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:13:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:12.222828+00:00 (in 59.999565 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:13:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:12 GMT)" (scheduled at 2025-11-04 09:13:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:13:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:13:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:13:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:13:12,371 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:13:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:13:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:29.351216+00:00 (in 14.999563 seconds) quotatotalworker stdout | 2025-11-04 09:13:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:29 GMT)" (scheduled at 2025-11-04 09:13:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:13:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:13:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:13:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:13:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:29 GMT)" executed successfully builder stdout | 2025-11-04 09:13:14,793 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:13:14,807 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:13:14,807 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:13:14,816 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:13:14,879 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:13:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:17.479080+00:00 (in 1.998867 seconds) securityworker stdout | 2025-11-04 09:13:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:45 GMT)" (scheduled at 2025-11-04 09:13:15.479769+00:00) securityworker stdout | 2025-11-04 09:13:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:13:15,496 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:15,496 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:15,496 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:15,496 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:15,496 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:15,500 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:15,501 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 15, 483827), 1, 2]) securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:15,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:15,505 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 15, 483827), 1, 2]) securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:15,508 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:13:15,508 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:21.530064+00:00 (in 5.994531 seconds) notificationworker stdout | 2025-11-04 09:13:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:15 GMT)" (scheduled at 2025-11-04 09:13:15.535095+00:00) notificationworker stdout | 2025-11-04 09:13:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:13:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:13:15,697 [286] [DEBUG] [app] Starting request: urn:request:dd2b54e2-6465-4879-9a45-17c9d2ef8abc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:15,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:15,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:15,701 [299] [DEBUG] [app] Starting request: urn:request:85a0f565-6bae-427d-9f4b-c58bdedf37e8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:15,702 [299] [DEBUG] [app] Ending request: urn:request:85a0f565-6bae-427d-9f4b-c58bdedf37e8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:85a0f565-6bae-427d-9f4b-c58bdedf37e8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:15,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:15,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:15,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:15,706 [287] [DEBUG] [app] Starting request: urn:request:1f84cdc1-de1a-4396-b197-f9e23baeee9c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:15,706 [287] [DEBUG] [app] Ending request: urn:request:1f84cdc1-de1a-4396-b197-f9e23baeee9c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1f84cdc1-de1a-4396-b197-f9e23baeee9c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:15,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:15,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:13:15,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:15,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:15,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:15,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:15,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:15,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:15,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:15,730 [286] [DEBUG] [app] Ending request: urn:request:dd2b54e2-6465-4879-9a45-17c9d2ef8abc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:dd2b54e2-6465-4879-9a45-17c9d2ef8abc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:15,731 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:15,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:13:15,783 [288] [DEBUG] [app] Starting request: urn:request:e4a49c3e-479e-48eb-b0ae-167476cfae63 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:15,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:15,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:15,788 [299] [DEBUG] [app] Starting request: urn:request:54e7c780-4a97-4af7-af90-611d6380d2a9 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:15,788 [299] [DEBUG] [app] Ending request: urn:request:54e7c780-4a97-4af7-af90-611d6380d2a9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:54e7c780-4a97-4af7-af90-611d6380d2a9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:15,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:13:15,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:15,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:15,792 [285] [DEBUG] [app] Starting request: urn:request:910f6593-2ead-40eb-9459-b76d8b261d26 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:15,792 [285] [DEBUG] [app] Ending request: urn:request:910f6593-2ead-40eb-9459-b76d8b261d26 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:910f6593-2ead-40eb-9459-b76d8b261d26', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:15,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:15,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:15,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:15,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:15,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:15,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:15,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:15,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:15,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:15,816 [288] [DEBUG] [app] Ending request: urn:request:e4a49c3e-479e-48eb-b0ae-167476cfae63 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e4a49c3e-479e-48eb-b0ae-167476cfae63', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:15,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:15,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:13:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:27.122849+00:00 (in 9.999637 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:27 GMT)" (scheduled at 2025-11-04 09:13:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 17, 123466), True, datetime.datetime(2025, 11, 4, 9, 13, 17, 123466), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:17,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:13:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:45.479769+00:00 (in 28.000291 seconds) securityworker stdout | 2025-11-04 09:13:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:47 GMT)" (scheduled at 2025-11-04 09:13:17.479080+00:00) securityworker stdout | 2025-11-04 09:13:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:17,480 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:17,482 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:13:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:13:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:13:17,812 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:18,471 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:13:19,797 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:13:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:13:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:51.226294+00:00 (in 29.999563 seconds) autopruneworker stdout | 2025-11-04 09:13:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:51 GMT)" (scheduled at 2025-11-04 09:13:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:13:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247541234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:13:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:13:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:13:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:13:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:13:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:32.435813+00:00 (in 11.000933 seconds) namespacegcworker stdout | 2025-11-04 09:13:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:21 GMT)" (scheduled at 2025-11-04 09:13:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:13:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:13:21,440 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 21, 435114), True, datetime.datetime(2025, 11, 4, 9, 13, 21, 435114), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:13:21,452 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:13:21,452 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:13:21,452 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:31.530064+00:00 (in 9.999600 seconds) notificationworker stdout | 2025-11-04 09:13:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:31 GMT)" (scheduled at 2025-11-04 09:13:21.530064+00:00) notificationworker stdout | 2025-11-04 09:13:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 21, 530753), True, datetime.datetime(2025, 11, 4, 9, 13, 21, 530753), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:13:22,056 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:13:24,817 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:13:24,830 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:13:24,831 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:13:24,841 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:24,912 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:37.122849+00:00 (in 9.999631 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:37 GMT)" (scheduled at 2025-11-04 09:13:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 27, 123480), True, datetime.datetime(2025, 11, 4, 9, 13, 27, 123480), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:37 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:13:27,152 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:13:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:13:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:57.222846+00:00 (in 29.999647 seconds) buildlogsarchiver stdout | 2025-11-04 09:13:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:57 GMT)" (scheduled at 2025-11-04 09:13:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:13:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 13, 27, 223414), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:13:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:13:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:13:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:13:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:13:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:13:27,864 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:13:28,519 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:13:28,901 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:29,220 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:13:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:13:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:44.351216+00:00 (in 14.999558 seconds) quotatotalworker stdout | 2025-11-04 09:13:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:44 GMT)" (scheduled at 2025-11-04 09:13:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:13:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:13:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:13:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:13:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:13:29,429 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:29,568 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:29,588 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:29,662 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:30,697 [286] [DEBUG] [app] Starting request: urn:request:aa855625-52d9-4413-b516-c6e2ac888801 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:30,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:30,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:30,701 [299] [DEBUG] [app] Starting request: urn:request:e7647173-9574-423e-bfcf-af879760a827 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:30,701 [299] [DEBUG] [app] Ending request: urn:request:e7647173-9574-423e-bfcf-af879760a827 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e7647173-9574-423e-bfcf-af879760a827', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:30,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:30,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:30,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:30,705 [288] [DEBUG] [app] Starting request: urn:request:68b0211a-46f9-442e-8df2-8a0a2482ae68 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:30,706 [288] [DEBUG] [app] Ending request: urn:request:68b0211a-46f9-442e-8df2-8a0a2482ae68 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:68b0211a-46f9-442e-8df2-8a0a2482ae68', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:30,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:30,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:30,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:30,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:30,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:30,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:30,725 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:30,729 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:30,732 [286] [DEBUG] [app] Ending request: urn:request:aa855625-52d9-4413-b516-c6e2ac888801 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:aa855625-52d9-4413-b516-c6e2ac888801', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:30,732 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.037) gunicorn-web stdout | 2025-11-04 09:13:30,732 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:13:30,784 [285] [DEBUG] [app] Starting request: urn:request:d282054c-3976-446b-a45a-c558e202eced (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:30,788 [299] [DEBUG] [app] Starting request: urn:request:0db96fe9-98be-49a2-8a1b-3af8058c06d8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:30,788 [299] [DEBUG] [app] Ending request: urn:request:0db96fe9-98be-49a2-8a1b-3af8058c06d8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0db96fe9-98be-49a2-8a1b-3af8058c06d8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:30,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:30,792 [287] [DEBUG] [app] Starting request: urn:request:8148e046-c36f-485a-be24-b80d6aa6ad50 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:30,792 [287] [DEBUG] [app] Ending request: urn:request:8148e046-c36f-485a-be24-b80d6aa6ad50 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8148e046-c36f-485a-be24-b80d6aa6ad50', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:30,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:30,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:30,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:30,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:30,817 [285] [DEBUG] [app] Ending request: urn:request:d282054c-3976-446b-a45a-c558e202eced (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d282054c-3976-446b-a45a-c558e202eced', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:30,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:13:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:13:31,156 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:13:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:41.530064+00:00 (in 9.999618 seconds) notificationworker stdout | 2025-11-04 09:13:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:41 GMT)" (scheduled at 2025-11-04 09:13:31.530064+00:00) notificationworker stdout | 2025-11-04 09:13:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 31, 530736), True, datetime.datetime(2025, 11, 4, 9, 13, 31, 530736), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:13:31,942 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:13:32,139 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:13:32,140 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:13:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:13:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:21.434460+00:00 (in 48.998258 seconds) namespacegcworker stdout | 2025-11-04 09:13:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:32 GMT)" (scheduled at 2025-11-04 09:13:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:13:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:13:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:13:33,433 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:13:33,434 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:13:33,537 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:13:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:13:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:48.625192+00:00 (in 15.001827 seconds) exportactionlogsworker stdout | 2025-11-04 09:13:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:33 GMT)" (scheduled at 2025-11-04 09:13:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:13:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:13:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 33, 623709), True, datetime.datetime(2025, 11, 4, 9, 13, 33, 623709), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:13:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:13:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:13:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:33 GMT)" executed successfully builder stdout | 2025-11-04 09:13:34,317 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:13:34,841 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:13:34,854 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:13:34,854 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:13:34,864 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:38.437104+00:00 (in 3.000902 seconds) gcworker stdout | 2025-11-04 09:13:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:05 GMT)" (scheduled at 2025-11-04 09:13:35.435809+00:00) gcworker stdout | 2025-11-04 09:13:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038015436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:13:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:13:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:13:35,612 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:13:35,981 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:13:36,609 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:13:37,003 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:40.124445+00:00 (in 3.001211 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:47 GMT)" (scheduled at 2025-11-04 09:13:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 37, 123512), True, datetime.datetime(2025, 11, 4, 9, 13, 37, 123512), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:13:37,896 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:37,903 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:37,935 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:37,943 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:37,970 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:37,979 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:38,003 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:38,128 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:38,145 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:38,367 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:38,371 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:38,375 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:13:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:05.435809+00:00 (in 26.998294 seconds) gcworker stdout | 2025-11-04 09:13:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:08 GMT)" (scheduled at 2025-11-04 09:13:38.437104+00:00) gcworker stdout | 2025-11-04 09:13:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:13:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247558449, None, 1, 0]) gcworker stdout | 2025-11-04 09:13:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:13:39,172 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:13:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:13:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:01.922827+00:00 (in 21.997779 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:13:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:39 GMT)" (scheduled at 2025-11-04 09:13:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:13:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:13:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:13:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:47.122849+00:00 (in 6.997995 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:40 GMT)" (scheduled at 2025-11-04 09:13:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:13:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:13:40,942 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:13:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:51.530064+00:00 (in 9.999508 seconds) notificationworker stdout | 2025-11-04 09:13:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:51 GMT)" (scheduled at 2025-11-04 09:13:41.530064+00:00) notificationworker stdout | 2025-11-04 09:13:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 41, 530865), True, datetime.datetime(2025, 11, 4, 9, 13, 41, 530865), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:13:42,380 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:13:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:13:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:57.523915+00:00 (in 14.090491 seconds) repositorygcworker stdout | 2025-11-04 09:13:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:43 GMT)" (scheduled at 2025-11-04 09:13:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:13:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:13:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 43, 433739), True, datetime.datetime(2025, 11, 4, 9, 13, 43, 433739), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:13:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:13:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:13:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:13:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:13:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:59.351216+00:00 (in 14.999611 seconds) quotatotalworker stdout | 2025-11-04 09:13:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:59 GMT)" (scheduled at 2025-11-04 09:13:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:13:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:13:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:13:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:13:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:13:59 GMT)" executed successfully builder stdout | 2025-11-04 09:13:44,864 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:13:44,877 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:13:44,877 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:13:44,887 [66] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:13:44,888 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:13:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:47.479080+00:00 (in 1.998852 seconds) securityworker stdout | 2025-11-04 09:13:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:15 GMT)" (scheduled at 2025-11-04 09:13:45.479769+00:00) securityworker stdout | 2025-11-04 09:13:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:45,481 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 45, 484060), 1, 2]) securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 45, 484060), 1, 2]) securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:13:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:13:45,697 [285] [DEBUG] [app] Starting request: urn:request:cead9e24-e082-452d-9083-f35e620e0c4b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:45,701 [299] [DEBUG] [app] Starting request: urn:request:556906e0-4b14-4357-b433-f3cbc96bae8c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:45,701 [299] [DEBUG] [app] Ending request: urn:request:556906e0-4b14-4357-b433-f3cbc96bae8c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:556906e0-4b14-4357-b433-f3cbc96bae8c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:45,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:45,706 [287] [DEBUG] [app] Starting request: urn:request:186bfc28-080c-47ca-9442-4508178d09fb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:45,706 [287] [DEBUG] [app] Ending request: urn:request:186bfc28-080c-47ca-9442-4508178d09fb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:186bfc28-080c-47ca-9442-4508178d09fb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:45,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:45,730 [285] [DEBUG] [app] Ending request: urn:request:cead9e24-e082-452d-9083-f35e620e0c4b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:cead9e24-e082-452d-9083-f35e620e0c4b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:45,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:13:45,784 [288] [DEBUG] [app] Starting request: urn:request:8a5651dd-ba74-4a0b-abf2-9860fcdd0927 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:13:45,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:45,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:13:45,788 [299] [DEBUG] [app] Starting request: urn:request:839de285-aea7-40fa-a95f-b388cbb76ce5 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:13:45,789 [299] [DEBUG] [app] Ending request: urn:request:839de285-aea7-40fa-a95f-b388cbb76ce5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:839de285-aea7-40fa-a95f-b388cbb76ce5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:45,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:45,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:45,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:45,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:13:45,793 [285] [DEBUG] [app] Starting request: urn:request:8317e30e-937f-42ed-95ee-008ee66f0b90 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:13:45,793 [285] [DEBUG] [app] Ending request: urn:request:8317e30e-937f-42ed-95ee-008ee66f0b90 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8317e30e-937f-42ed-95ee-008ee66f0b90', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:13:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:45,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:13:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:13:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:45,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:45,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:45,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:45,817 [288] [DEBUG] [app] Ending request: urn:request:8a5651dd-ba74-4a0b-abf2-9860fcdd0927 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8a5651dd-ba74-4a0b-abf2-9860fcdd0927', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:45,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:13:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:13:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:13:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:57.122849+00:00 (in 9.999580 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:57 GMT)" (scheduled at 2025-11-04 09:13:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 47, 123612), True, datetime.datetime(2025, 11, 4, 9, 13, 47, 123612), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:13:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:15.479769+00:00 (in 28.000291 seconds) securityworker stdout | 2025-11-04 09:13:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:17 GMT)" (scheduled at 2025-11-04 09:13:47.479080+00:00) securityworker stdout | 2025-11-04 09:13:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:13:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 47, 482861), 1, 2]) securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:13:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 47, 482861), 1, 2]) securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:13:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:13:47,831 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:48,480 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:13:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:13:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:33.622985+00:00 (in 44.997396 seconds) exportactionlogsworker stdout | 2025-11-04 09:13:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:48 GMT)" (scheduled at 2025-11-04 09:13:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:13:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:13:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:13:49,807 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:13:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:13:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:21.226294+00:00 (in 29.999552 seconds) autopruneworker stdout | 2025-11-04 09:13:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:21 GMT)" (scheduled at 2025-11-04 09:13:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:13:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247571234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:13:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:13:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:13:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:01.530064+00:00 (in 9.999584 seconds) notificationworker stdout | 2025-11-04 09:13:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:01 GMT)" (scheduled at 2025-11-04 09:13:51.530064+00:00) notificationworker stdout | 2025-11-04 09:13:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 51, 530748), True, datetime.datetime(2025, 11, 4, 9, 13, 51, 530748), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:13:52,065 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:13:54,888 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:13:54,901 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:13:54,901 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:13:54,911 [66] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:54,931 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:07.122849+00:00 (in 9.999638 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:07 GMT)" (scheduled at 2025-11-04 09:13:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 57, 123533), True, datetime.datetime(2025, 11, 4, 9, 13, 57, 123533), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:13:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:09.133645+00:00 (in 12.001442 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:57 GMT)" (scheduled at 2025-11-04 09:13:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:57,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:57,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:57,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:13:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:13:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:13:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:13:57,161 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:13:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:13:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:27.222846+00:00 (in 29.999600 seconds) buildlogsarchiver stdout | 2025-11-04 09:13:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:27 GMT)" (scheduled at 2025-11-04 09:13:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:13:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 13, 57, 223469), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:13:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:13:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:13:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:13:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:13:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:13:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:13:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:02.523412+00:00 (in 4.998715 seconds) repositorygcworker stdout | 2025-11-04 09:13:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:57 GMT)" (scheduled at 2025-11-04 09:13:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:13:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:13:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:13:57,873 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:13:58,527 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:13:58,910 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:59,231 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:13:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:13:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:14.351216+00:00 (in 14.999560 seconds) quotatotalworker stdout | 2025-11-04 09:13:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:14 GMT)" (scheduled at 2025-11-04 09:13:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:13:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:13:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:13:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:13:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:13:59,441 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:59,582 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:59,602 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:59,676 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:00,697 [285] [DEBUG] [app] Starting request: urn:request:4f6277f0-8f34-4637-bd6f-a9df7626fc54 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:00,702 [292] [DEBUG] [app] Starting request: urn:request:e0c713ca-0688-40fe-b655-60560506bf53 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:00,702 [292] [DEBUG] [app] Ending request: urn:request:e0c713ca-0688-40fe-b655-60560506bf53 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e0c713ca-0688-40fe-b655-60560506bf53', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:14:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:00,706 [288] [DEBUG] [app] Starting request: urn:request:44040db1-d29d-4146-ba30-b6debf4414cb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:00,706 [288] [DEBUG] [app] Ending request: urn:request:44040db1-d29d-4146-ba30-b6debf4414cb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:44040db1-d29d-4146-ba30-b6debf4414cb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:00,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:00,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:00,730 [285] [DEBUG] [app] Ending request: urn:request:4f6277f0-8f34-4637-bd6f-a9df7626fc54 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4f6277f0-8f34-4637-bd6f-a9df7626fc54', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:00,730 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:14:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:14:00,783 [287] [DEBUG] [app] Starting request: urn:request:daa2841d-7073-4ba6-843e-f516c19db4ec (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:00,784 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:00,788 [292] [DEBUG] [app] Starting request: urn:request:6d646c14-f0be-4ebe-8ea3-98ab96723bbc (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:00,788 [292] [DEBUG] [app] Ending request: urn:request:6d646c14-f0be-4ebe-8ea3-98ab96723bbc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6d646c14-f0be-4ebe-8ea3-98ab96723bbc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:14:00,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:14:00,788 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:00,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:00,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:00,792 [285] [DEBUG] [app] Starting request: urn:request:b3254048-9959-46ee-b7cd-ada9f06ce1b4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:00,792 [285] [DEBUG] [app] Ending request: urn:request:b3254048-9959-46ee-b7cd-ada9f06ce1b4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b3254048-9959-46ee-b7cd-ada9f06ce1b4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:00,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:00,792 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:00,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:00,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:00,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:00,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:00,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:00,809 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:00,812 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:00,815 [287] [DEBUG] [app] Ending request: urn:request:daa2841d-7073-4ba6-843e-f516c19db4ec (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:daa2841d-7073-4ba6-843e-f516c19db4ec', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:00,815 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.033) gunicorn-web stdout | 2025-11-04 09:14:00,816 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:14:01,165 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:11.530064+00:00 (in 9.999540 seconds) notificationworker stdout | 2025-11-04 09:14:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:11 GMT)" (scheduled at 2025-11-04 09:14:01.530064+00:00) notificationworker stdout | 2025-11-04 09:14:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 1, 530779), True, datetime.datetime(2025, 11, 4, 9, 14, 1, 530779), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:14:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:14:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:39.924652+00:00 (in 38.001407 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:14:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:01 GMT)" (scheduled at 2025-11-04 09:14:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:14:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:14:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 1, 923483), True, datetime.datetime(2025, 11, 4, 9, 14, 1, 923483), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:14:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:14:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:14:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:14:01,951 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:14:02,148 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:14:02,149 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:14:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:14:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:43.432867+00:00 (in 40.909005 seconds) repositorygcworker stdout | 2025-11-04 09:14:02,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:02 GMT)" (scheduled at 2025-11-04 09:14:02.523412+00:00) repositorygcworker stdout | 2025-11-04 09:14:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 14, 2, 524132), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:14:02,536 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 2, 524132), True, datetime.datetime(2025, 11, 4, 9, 14, 2, 524132), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:14:02,539 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 2, 524132), True, datetime.datetime(2025, 11, 4, 9, 14, 2, 524132), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 14, 2, 524132), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:14:02,542 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:14:02,543 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:02 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:14:03,442 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:14:03,443 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:14:03,546 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:04,328 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:04,911 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:14:04,924 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:14:04,925 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:14:04,944 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:08.437104+00:00 (in 3.000893 seconds) gcworker stdout | 2025-11-04 09:14:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:35 GMT)" (scheduled at 2025-11-04 09:14:05.435809+00:00) gcworker stdout | 2025-11-04 09:14:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038045436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:14:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:14:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:14:05,621 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:14:05,989 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:14:06,618 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:14:07,012 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:17.122849+00:00 (in 9.999626 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:17 GMT)" (scheduled at 2025-11-04 09:14:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 7, 123459), True, datetime.datetime(2025, 11, 4, 9, 14, 7, 123459), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:14:07,911 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:07,916 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:07,945 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:07,957 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:07,982 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:07,994 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:08,014 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:08,140 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:08,157 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:08,378 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:08,386 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:08,390 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:14:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:35.435809+00:00 (in 26.998301 seconds) gcworker stdout | 2025-11-04 09:14:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:38 GMT)" (scheduled at 2025-11-04 09:14:08.437104+00:00) gcworker stdout | 2025-11-04 09:14:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:14:08,451 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247588450, None, 1, 0]) gcworker stdout | 2025-11-04 09:14:08,454 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:08,455 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:38 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:14:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:14:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:57.131795+00:00 (in 47.997717 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:09 GMT)" (scheduled at 2025-11-04 09:14:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:14:09,147 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:14:09,147 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:14:09,180 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:14:10,951 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:11,531 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:11,531 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:15.535095+00:00 (in 4.003779 seconds) notificationworker stdout | 2025-11-04 09:14:11,531 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:21 GMT)" (scheduled at 2025-11-04 09:14:11.530064+00:00) notificationworker stdout | 2025-11-04 09:14:11,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:11,532 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 11, 531617), True, datetime.datetime(2025, 11, 4, 9, 14, 11, 531617), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:14:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:14:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:12.222828+00:00 (in 59.999536 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:14:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:12 GMT)" (scheduled at 2025-11-04 09:14:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:14:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:14:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:14:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:14:12,395 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:14:13,539 [103] [DEBUG] [apscheduler.scheduler] Looking for jobs to run servicekey stdout | 2025-11-04 09:14:13,540 [103] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 10:09:13.539760+00:00 (in 3299.999528 seconds) servicekey stdout | 2025-11-04 09:14:13,540 [103] [INFO] [apscheduler.executors.default] Running job "ServiceKeyWorker._refresh_service_key (trigger: interval[0:55:00], next run at: 2025-11-04 10:09:13 GMT)" (scheduled at 2025-11-04 09:14:13.539760+00:00) servicekey stdout | 2025-11-04 09:14:13,540 [103] [DEBUG] [__main__] Starting automatic refresh of service key oSDdxxw3mt0ej3ElGg9LBcL2koxWBtCnAyvelg76mY8 to new expiration 2025-11-04 11:14:13.540477 servicekey stdout | 2025-11-04 09:14:13,541 [103] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE (("t1"."kid" = %s) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s))) LIMIT %s OFFSET %s', ['oSDdxxw3mt0ej3ElGg9LBcL2koxWBtCnAyvelg76mY8', datetime.datetime(2025, 10, 28, 9, 14, 13, 540730), None, 1, 0]) servicekey stdout | 2025-11-04 09:14:13,553 [103] [DEBUG] [peewee] ('UPDATE "servicekey" SET "name" = %s, "kid" = %s, "service" = %s, "jwk" = %s, "metadata" = %s, "created_date" = %s, "expiration_date" = %s, "rotation_duration" = %s, "approval_id" = %s WHERE ("servicekey"."id" = %s)', ['https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com:443', 'oSDdxxw3mt0ej3ElGg9LBcL2koxWBtCnAyvelg76mY8', 'quay', '{"n": "m2sdBJ_AQs8mSmKygaUn8jeXjQn4hSLv7OJ_u6qckRSe6tvlmhLfyZ1op0d8EVih8Fj8n6monhxTnCOXhS7MtmqVBGQQFldWDUJghDJdW8fTUVwoCR4-yqC9zv8nzwxPmX8ZiEz5IfFVPRLoeP8CgrppGBjFMWDt-Sl8ihyaHu8I17MXAkSpyh1_U0QLIssjWv4nghbct7lGwjQJWLCIroEs_Iwe0bE6OeMp7J3aXYEAUaniftSqt5f_CEujJWx17ueeYlrfLIbOKD1JNZT686pq5GsnHcOcyk78hITVfwBLXfHiRp3ocmcv3W19rtR25zbXnHt5cWCPt2GLyHHBMQ", "e": "AQAB", "kty": "RSA", "kid": "oSDdxxw3mt0ej3ElGg9LBcL2koxWBtCnAyvelg76mY8"}', '{"created_by": "CLI tool"}', datetime.datetime(2025, 11, 4, 6, 40, 21, 74583), datetime.datetime(2025, 11, 4, 11, 14, 13, 540477), None, 4, 4]) servicekey stdout | 2025-11-04 09:14:13,557 [103] [DEBUG] [__main__] Finished automatic refresh of service key oSDdxxw3mt0ej3ElGg9LBcL2koxWBtCnAyvelg76mY8 with new expiration 2025-11-04 11:14:13.540477 servicekey stdout | 2025-11-04 09:14:13,557 [103] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:14:13,557 [103] [INFO] [apscheduler.executors.default] Job "ServiceKeyWorker._refresh_service_key (trigger: interval[0:55:00], next run at: 2025-11-04 10:09:13 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:14:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:14:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:29.351216+00:00 (in 14.999556 seconds) quotatotalworker stdout | 2025-11-04 09:14:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:29 GMT)" (scheduled at 2025-11-04 09:14:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:14:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:14:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:14:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:14:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:14:14,897 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:14,944 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:14:14,958 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:14:14,958 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:14:14,968 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:14:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:17.479080+00:00 (in 1.998845 seconds) securityworker stdout | 2025-11-04 09:14:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:45 GMT)" (scheduled at 2025-11-04 09:14:15.479769+00:00) securityworker stdout | 2025-11-04 09:14:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 15, 483911), 1, 2]) securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 15, 483911), 1, 2]) securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:21.530064+00:00 (in 5.994548 seconds) notificationworker stdout | 2025-11-04 09:14:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:15 GMT)" (scheduled at 2025-11-04 09:14:15.535095+00:00) notificationworker stdout | 2025-11-04 09:14:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:14:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:14:15,697 [285] [DEBUG] [app] Starting request: urn:request:e32d922d-506a-4c72-baf5-c30db88967a9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:15,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:15,702 [299] [DEBUG] [app] Starting request: urn:request:28cdad7b-8461-454d-9829-0d763c2ae908 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:15,702 [299] [DEBUG] [app] Ending request: urn:request:28cdad7b-8461-454d-9829-0d763c2ae908 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:28cdad7b-8461-454d-9829-0d763c2ae908', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:14:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:14:15,703 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:15,706 [288] [DEBUG] [app] Starting request: urn:request:c22668af-9c3e-4215-be79-7a37ec4f97fa (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:15,706 [288] [DEBUG] [app] Ending request: urn:request:c22668af-9c3e-4215-be79-7a37ec4f97fa (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c22668af-9c3e-4215-be79-7a37ec4f97fa', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:15,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:15,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:15,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:15,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:15,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:15,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:15,731 [285] [DEBUG] [app] Ending request: urn:request:e32d922d-506a-4c72-baf5-c30db88967a9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e32d922d-506a-4c72-baf5-c30db88967a9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:15,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:15,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:14:15,783 [285] [DEBUG] [app] Starting request: urn:request:9c14f95a-f3b1-405b-9f6b-2274cfa71a4d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:15,785 [292] [DEBUG] [app] Starting request: urn:request:39c01886-7ea5-4137-b0dc-f3f399832db4 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:15,785 [292] [DEBUG] [app] Ending request: urn:request:39c01886-7ea5-4137-b0dc-f3f399832db4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:39c01886-7ea5-4137-b0dc-f3f399832db4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:14:15,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:15,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:15,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:15,789 [288] [DEBUG] [app] Starting request: urn:request:6e7edfde-cc70-4473-9380-3bda6d289034 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:15,789 [288] [DEBUG] [app] Ending request: urn:request:6e7edfde-cc70-4473-9380-3bda6d289034 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6e7edfde-cc70-4473-9380-3bda6d289034', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:15,813 [285] [DEBUG] [app] Ending request: urn:request:9c14f95a-f3b1-405b-9f6b-2274cfa71a4d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9c14f95a-f3b1-405b-9f6b-2274cfa71a4d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:15,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:14:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:27.122849+00:00 (in 9.999623 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:27 GMT)" (scheduled at 2025-11-04 09:14:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 17, 123467), True, datetime.datetime(2025, 11, 4, 9, 14, 17, 123467), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:14:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:45.479769+00:00 (in 28.000189 seconds) securityworker stdout | 2025-11-04 09:14:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:47 GMT)" (scheduled at 2025-11-04 09:14:17.479080+00:00) securityworker stdout | 2025-11-04 09:14:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:14:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:14:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:14:17,840 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:18,498 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:14:19,816 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:14:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:14:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:51.226294+00:00 (in 29.999623 seconds) autopruneworker stdout | 2025-11-04 09:14:21,227 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:51 GMT)" (scheduled at 2025-11-04 09:14:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:14:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247601235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:14:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:14:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:14:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:14:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:14:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:32.435813+00:00 (in 11.000942 seconds) namespacegcworker stdout | 2025-11-04 09:14:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:21 GMT)" (scheduled at 2025-11-04 09:14:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:14:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:14:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 21, 435165), True, datetime.datetime(2025, 11, 4, 9, 14, 21, 435165), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:14:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:14:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:14:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:31.530064+00:00 (in 9.999590 seconds) notificationworker stdout | 2025-11-04 09:14:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:31 GMT)" (scheduled at 2025-11-04 09:14:21.530064+00:00) notificationworker stdout | 2025-11-04 09:14:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 21, 530757), True, datetime.datetime(2025, 11, 4, 9, 14, 21, 530757), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:14:22,074 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:24,941 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:24,968 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:14:24,981 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:14:24,981 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:14:24,991 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:37.122849+00:00 (in 9.999632 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:37 GMT)" (scheduled at 2025-11-04 09:14:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 27, 123457), True, datetime.datetime(2025, 11, 4, 9, 14, 27, 123457), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:37 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:14:27,170 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:14:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:14:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:57.222846+00:00 (in 29.999599 seconds) buildlogsarchiver stdout | 2025-11-04 09:14:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:57 GMT)" (scheduled at 2025-11-04 09:14:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:14:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 14, 27, 223461), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:14:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:14:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:14:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:14:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:14:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:14:27,882 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:14:28,536 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:14:28,918 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:29,241 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:14:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:14:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:44.351216+00:00 (in 14.999551 seconds) quotatotalworker stdout | 2025-11-04 09:14:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:44 GMT)" (scheduled at 2025-11-04 09:14:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:14:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:14:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:14:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:14:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:14:29,453 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:29,596 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:29,615 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:29,690 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:30,697 [288] [DEBUG] [app] Starting request: urn:request:c5ac828b-1c18-437c-adf0-d4b00823284b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:30,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:30,701 [299] [DEBUG] [app] Starting request: urn:request:62a09443-cc16-444e-9cc1-936ee3341d12 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:30,701 [299] [DEBUG] [app] Ending request: urn:request:62a09443-cc16-444e-9cc1-936ee3341d12 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:62a09443-cc16-444e-9cc1-936ee3341d12', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:14:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:30,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:30,705 [285] [DEBUG] [app] Starting request: urn:request:dd8609b2-d01a-4e25-9f50-889f66f0c8b2 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:30,706 [285] [DEBUG] [app] Ending request: urn:request:dd8609b2-d01a-4e25-9f50-889f66f0c8b2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dd8609b2-d01a-4e25-9f50-889f66f0c8b2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:14:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:30,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:30,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:30,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:30,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:30,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:30,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:30,731 [288] [DEBUG] [app] Ending request: urn:request:c5ac828b-1c18-437c-adf0-d4b00823284b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c5ac828b-1c18-437c-adf0-d4b00823284b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:30,731 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:14:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:14:30,783 [285] [DEBUG] [app] Starting request: urn:request:dfc386d0-47e0-4306-84a2-c9fc91debf6f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:30,787 [292] [DEBUG] [app] Starting request: urn:request:c4a1c08e-0d0d-4d1c-a085-ded0104edd44 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:30,788 [292] [DEBUG] [app] Ending request: urn:request:c4a1c08e-0d0d-4d1c-a085-ded0104edd44 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c4a1c08e-0d0d-4d1c-a085-ded0104edd44', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:14:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:30,792 [288] [DEBUG] [app] Starting request: urn:request:5552b42e-697e-4c8f-b4dd-d6ea39654ff9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:30,792 [288] [DEBUG] [app] Ending request: urn:request:5552b42e-697e-4c8f-b4dd-d6ea39654ff9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5552b42e-697e-4c8f-b4dd-d6ea39654ff9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:30,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:30,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:14:30,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:30,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:30,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:30,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:30,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:30,816 [285] [DEBUG] [app] Ending request: urn:request:dfc386d0-47e0-4306-84a2-c9fc91debf6f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:dfc386d0-47e0-4306-84a2-c9fc91debf6f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:30,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:31,174 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:40.533243+00:00 (in 9.002725 seconds) notificationworker stdout | 2025-11-04 09:14:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:41 GMT)" (scheduled at 2025-11-04 09:14:31.530064+00:00) notificationworker stdout | 2025-11-04 09:14:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 31, 530782), True, datetime.datetime(2025, 11, 4, 9, 14, 31, 530782), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:14:31,959 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:14:32,157 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:14:32,158 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:14:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:14:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:21.434460+00:00 (in 48.998249 seconds) namespacegcworker stdout | 2025-11-04 09:14:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:32 GMT)" (scheduled at 2025-11-04 09:14:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:14:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:14:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:14:33,451 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:14:33,451 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:14:33,554 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:14:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:14:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:48.625192+00:00 (in 15.001780 seconds) exportactionlogsworker stdout | 2025-11-04 09:14:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:33 GMT)" (scheduled at 2025-11-04 09:14:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:14:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:14:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 33, 623745), True, datetime.datetime(2025, 11, 4, 9, 14, 33, 623745), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:14:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:14:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:14:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:33 GMT)" executed successfully builder stdout | 2025-11-04 09:14:34,337 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:34,991 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:14:35,005 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:14:35,005 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:14:35,015 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:38.437104+00:00 (in 3.000913 seconds) gcworker stdout | 2025-11-04 09:14:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:05 GMT)" (scheduled at 2025-11-04 09:14:35.435809+00:00) gcworker stdout | 2025-11-04 09:14:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038075436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:14:35,449 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:14:35,449 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:14:35,630 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:14:35,998 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:14:36,628 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:14:37,020 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:40.124445+00:00 (in 3.001211 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:47 GMT)" (scheduled at 2025-11-04 09:14:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 37, 123519), True, datetime.datetime(2025, 11, 4, 9, 14, 37, 123519), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:14:37,924 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:37,928 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:37,956 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:37,970 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:37,994 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:38,008 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:38,027 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:38,155 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:38,168 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:38,388 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:38,400 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:38,404 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:14:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:05.435809+00:00 (in 26.998344 seconds) gcworker stdout | 2025-11-04 09:14:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:08 GMT)" (scheduled at 2025-11-04 09:14:38.437104+00:00) gcworker stdout | 2025-11-04 09:14:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:14:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247618449, None, 1, 0]) gcworker stdout | 2025-11-04 09:14:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:14:39,189 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:14:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:14:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:01.922827+00:00 (in 21.997723 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:14:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:39 GMT)" (scheduled at 2025-11-04 09:14:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:14:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:14:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:14:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:47.122849+00:00 (in 6.998028 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:40 GMT)" (scheduled at 2025-11-04 09:14:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:14:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:41.530064+00:00 (in 0.996408 seconds) notificationworker stdout | 2025-11-04 09:14:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:40 GMT)" (scheduled at 2025-11-04 09:14:40.533243+00:00) notificationworker stdout | 2025-11-04 09:14:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 14, 40, 533885), 'notification/%']) notificationworker stdout | 2025-11-04 09:14:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 40, 533885), True, datetime.datetime(2025, 11, 4, 9, 14, 40, 533885), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:14:40,550 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 40, 533885), True, datetime.datetime(2025, 11, 4, 9, 14, 40, 533885), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 14, 40, 533885), 'notification/%']) notificationworker stdout | 2025-11-04 09:14:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:14:40,960 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:51.530064+00:00 (in 9.999554 seconds) notificationworker stdout | 2025-11-04 09:14:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:51 GMT)" (scheduled at 2025-11-04 09:14:41.530064+00:00) notificationworker stdout | 2025-11-04 09:14:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 41, 530803), True, datetime.datetime(2025, 11, 4, 9, 14, 41, 530803), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:14:42,403 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:14:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:14:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:57.523915+00:00 (in 14.090657 seconds) repositorygcworker stdout | 2025-11-04 09:14:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:43 GMT)" (scheduled at 2025-11-04 09:14:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:14:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:14:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 43, 433521), True, datetime.datetime(2025, 11, 4, 9, 14, 43, 433521), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:14:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:14:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:14:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:14:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:14:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:59.351216+00:00 (in 14.999543 seconds) quotatotalworker stdout | 2025-11-04 09:14:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:59 GMT)" (scheduled at 2025-11-04 09:14:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:14:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:14:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:14:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:14:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:14:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:14:44,905 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:45,015 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:14:45,029 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:14:45,029 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:14:45,039 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:14:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:47.479080+00:00 (in 1.998880 seconds) securityworker stdout | 2025-11-04 09:14:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:15 GMT)" (scheduled at 2025-11-04 09:14:45.479769+00:00) securityworker stdout | 2025-11-04 09:14:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:14:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 45, 483753), 1, 2]) securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 45, 483753), 1, 2]) securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:14:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:15 GMT)" executed successfully securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker gunicorn-web stdout | 2025-11-04 09:14:45,696 [285] [DEBUG] [app] Starting request: urn:request:fae1d538-3f82-49a7-b3e7-1d64c3226aa2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:45,701 [299] [DEBUG] [app] Starting request: urn:request:05566b03-af10-4b08-a8aa-0a7f3f94940f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:45,701 [299] [DEBUG] [app] Ending request: urn:request:05566b03-af10-4b08-a8aa-0a7f3f94940f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:05566b03-af10-4b08-a8aa-0a7f3f94940f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:14:45,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:14:45,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:45,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:45,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:45,705 [287] [DEBUG] [app] Starting request: urn:request:b6edab77-0019-4b79-8215-0ce88839f60d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:45,705 [287] [DEBUG] [app] Ending request: urn:request:b6edab77-0019-4b79-8215-0ce88839f60d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b6edab77-0019-4b79-8215-0ce88839f60d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:45,705 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:45,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:45,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:45,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:45,728 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:45,731 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:45,734 [285] [DEBUG] [app] Ending request: urn:request:fae1d538-3f82-49a7-b3e7-1d64c3226aa2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fae1d538-3f82-49a7-b3e7-1d64c3226aa2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:45,734 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:45,734 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.039 47 0.039) gunicorn-web stdout | 2025-11-04 09:14:45,784 [287] [DEBUG] [app] Starting request: urn:request:43003eb0-0030-4121-9d7f-0685e86c7f17 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:14:45,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:45,788 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:14:45,788 [299] [DEBUG] [app] Starting request: urn:request:25479f51-4cfb-4e06-9d29-a5affca1b1b1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:14:45,789 [299] [DEBUG] [app] Ending request: urn:request:25479f51-4cfb-4e06-9d29-a5affca1b1b1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:25479f51-4cfb-4e06-9d29-a5affca1b1b1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:14:45,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:45,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:45,790 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:45,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:14:45,793 [288] [DEBUG] [app] Starting request: urn:request:d21f7c80-6e67-4fad-aa25-962c3da3c1aa (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:14:45,793 [288] [DEBUG] [app] Ending request: urn:request:d21f7c80-6e67-4fad-aa25-962c3da3c1aa (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d21f7c80-6e67-4fad-aa25-962c3da3c1aa', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:14:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:45,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:14:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:45,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:45,794 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:45,794 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:45,794 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:45,802 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:45,802 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:45,812 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:45,815 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:45,818 [287] [DEBUG] [app] Ending request: urn:request:43003eb0-0030-4121-9d7f-0685e86c7f17 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:43003eb0-0030-4121-9d7f-0685e86c7f17', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:45,818 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:45,818 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:14:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:14:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:14:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:53.124163+00:00 (in 6.000911 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:57 GMT)" (scheduled at 2025-11-04 09:14:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 47, 123542), True, datetime.datetime(2025, 11, 4, 9, 14, 47, 123542), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:14:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:15.479769+00:00 (in 28.000184 seconds) securityworker stdout | 2025-11-04 09:14:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:17 GMT)" (scheduled at 2025-11-04 09:14:47.479080+00:00) securityworker stdout | 2025-11-04 09:14:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:14:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:14:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 47, 483335), 1, 2]) securityworker stdout | 2025-11-04 09:14:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 47, 483335), 1, 2]) securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:14:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:14:47,849 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:48,507 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:14:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:14:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:33.622985+00:00 (in 44.997359 seconds) exportactionlogsworker stdout | 2025-11-04 09:14:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:48 GMT)" (scheduled at 2025-11-04 09:14:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:14:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:14:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:14:49,825 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:14:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:14:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:21.226294+00:00 (in 29.999601 seconds) autopruneworker stdout | 2025-11-04 09:14:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:21 GMT)" (scheduled at 2025-11-04 09:14:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:14:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247631234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:14:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:14:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:14:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:01.530064+00:00 (in 9.999598 seconds) notificationworker stdout | 2025-11-04 09:14:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:01 GMT)" (scheduled at 2025-11-04 09:14:51.530064+00:00) notificationworker stdout | 2025-11-04 09:14:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 51, 530761), True, datetime.datetime(2025, 11, 4, 9, 14, 51, 530761), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:51,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:14:52,083 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:57.122849+00:00 (in 3.998262 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:53 GMT)" (scheduled at 2025-11-04 09:14:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 14, 53, 124846), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:14:53,137 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 53, 124846), True, datetime.datetime(2025, 11, 4, 9, 14, 53, 124846), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:14:53,141 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 53, 124846), True, datetime.datetime(2025, 11, 4, 9, 14, 53, 124846), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 14, 53, 124846), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:14:53,144 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:53,144 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:54,950 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:55,039 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:14:55,052 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:14:55,052 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:14:55,063 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:07.122849+00:00 (in 9.999636 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:07 GMT)" (scheduled at 2025-11-04 09:14:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 57, 123458), True, datetime.datetime(2025, 11, 4, 9, 14, 57, 123458), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:14:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:09.133645+00:00 (in 12.001450 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:57 GMT)" (scheduled at 2025-11-04 09:14:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:57,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:14:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:14:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:14:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:14:57,178 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:14:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:14:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:27.222846+00:00 (in 29.999558 seconds) buildlogsarchiver stdout | 2025-11-04 09:14:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:27 GMT)" (scheduled at 2025-11-04 09:14:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:14:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 14, 57, 223507), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:14:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:14:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:14:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:14:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:14:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:14:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:14:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:43.432867+00:00 (in 45.908573 seconds) repositorygcworker stdout | 2025-11-04 09:14:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:57 GMT)" (scheduled at 2025-11-04 09:14:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:14:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:14:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:14:57,890 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:14:58,544 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:14:58,927 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:59,251 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:14:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:14:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:14.351216+00:00 (in 14.999521 seconds) quotatotalworker stdout | 2025-11-04 09:14:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:14 GMT)" (scheduled at 2025-11-04 09:14:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:14:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:14:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:14:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:14:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:14:59,465 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:59,609 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:59,629 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:59,703 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:00,696 [288] [DEBUG] [app] Starting request: urn:request:09151841-4b78-4725-b759-462465a9473e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:00,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:00,701 [299] [DEBUG] [app] Starting request: urn:request:aa1b98c4-6588-4132-b2c5-5c2073653b87 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:00,701 [299] [DEBUG] [app] Ending request: urn:request:aa1b98c4-6588-4132-b2c5-5c2073653b87 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:aa1b98c4-6588-4132-b2c5-5c2073653b87', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:15:00,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:00,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:00,705 [285] [DEBUG] [app] Starting request: urn:request:5a2182ca-8000-49c0-9cd2-0e3a8d6198be (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:00,706 [285] [DEBUG] [app] Ending request: urn:request:5a2182ca-8000-49c0-9cd2-0e3a8d6198be (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5a2182ca-8000-49c0-9cd2-0e3a8d6198be', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:00,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:00,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:00,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:00,730 [288] [DEBUG] [app] Ending request: urn:request:09151841-4b78-4725-b759-462465a9473e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:09151841-4b78-4725-b759-462465a9473e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:00,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:15:00,783 [285] [DEBUG] [app] Starting request: urn:request:44bf9868-c0f3-4bc7-b936-173c5f73e429 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:00,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:00,787 [299] [DEBUG] [app] Starting request: urn:request:a2e3b237-f8b8-4e61-b9eb-fc4ab10b48a2 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:00,788 [299] [DEBUG] [app] Ending request: urn:request:a2e3b237-f8b8-4e61-b9eb-fc4ab10b48a2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a2e3b237-f8b8-4e61-b9eb-fc4ab10b48a2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:15:00,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:00,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:00,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:00,792 [288] [DEBUG] [app] Starting request: urn:request:6e514e98-7720-4e0f-99e9-8b70f6edf049 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:00,792 [288] [DEBUG] [app] Ending request: urn:request:6e514e98-7720-4e0f-99e9-8b70f6edf049 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6e514e98-7720-4e0f-99e9-8b70f6edf049', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:00,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:00,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:00,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:00,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:00,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:00,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:00,816 [285] [DEBUG] [app] Ending request: urn:request:44bf9868-c0f3-4bc7-b936-173c5f73e429 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:44bf9868-c0f3-4bc7-b936-173c5f73e429', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:00,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:00,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:01,183 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:11.530064+00:00 (in 9.999531 seconds) notificationworker stdout | 2025-11-04 09:15:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:11 GMT)" (scheduled at 2025-11-04 09:15:01.530064+00:00) notificationworker stdout | 2025-11-04 09:15:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 1, 530852), True, datetime.datetime(2025, 11, 4, 9, 15, 1, 530852), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:15:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:15:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:39.924652+00:00 (in 38.001392 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:15:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:01 GMT)" (scheduled at 2025-11-04 09:15:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:15:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:15:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 1, 923507), True, datetime.datetime(2025, 11, 4, 9, 15, 1, 923507), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:15:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:15:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:15:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:15:01,968 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:15:02,166 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:15:02,167 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:15:03,460 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:15:03,460 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:15:03,563 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:04,346 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:05,063 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:15:05,077 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:15:05,077 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:15:05,087 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:08.437104+00:00 (in 3.000899 seconds) gcworker stdout | 2025-11-04 09:15:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:35 GMT)" (scheduled at 2025-11-04 09:15:05.435809+00:00) gcworker stdout | 2025-11-04 09:15:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038105436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:15:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:15:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:15:05,638 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:15:06,007 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:15:06,638 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:15:07,029 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:17.122849+00:00 (in 9.999609 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:17 GMT)" (scheduled at 2025-11-04 09:15:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 7, 123514), True, datetime.datetime(2025, 11, 4, 9, 15, 7, 123514), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:15:07,938 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:07,938 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:07,966 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:07,982 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:08,006 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:08,018 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:08,041 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:08,168 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:08,186 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:08,398 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:08,415 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:08,418 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:15:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:35.435809+00:00 (in 26.998194 seconds) gcworker stdout | 2025-11-04 09:15:08,438 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:38 GMT)" (scheduled at 2025-11-04 09:15:08.437104+00:00) gcworker stdout | 2025-11-04 09:15:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:15:08,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247648450, None, 1, 0]) gcworker stdout | 2025-11-04 09:15:08,454 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:08,454 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:38 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:15:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:15:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:57.131795+00:00 (in 47.997715 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:09 GMT)" (scheduled at 2025-11-04 09:15:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:15:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:15:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:15:09,198 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:15:10,968 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:15.535095+00:00 (in 4.004563 seconds) notificationworker stdout | 2025-11-04 09:15:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:21 GMT)" (scheduled at 2025-11-04 09:15:11.530064+00:00) notificationworker stdout | 2025-11-04 09:15:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 11, 530811), True, datetime.datetime(2025, 11, 4, 9, 15, 11, 530811), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:15:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:15:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:12.222828+00:00 (in 59.999584 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:15:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:12 GMT)" (scheduled at 2025-11-04 09:15:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:15:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:15:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:15:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:15:12,412 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:15:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:15:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:29.351216+00:00 (in 14.999555 seconds) quotatotalworker stdout | 2025-11-04 09:15:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:29 GMT)" (scheduled at 2025-11-04 09:15:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:15:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:15:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:15:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:15:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:15:14,914 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:15,087 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:15:15,100 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:15:15,100 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:15:15,110 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:17.479080+00:00 (in 1.998816 seconds) securityworker stdout | 2025-11-04 09:15:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:45 GMT)" (scheduled at 2025-11-04 09:15:15.479769+00:00) securityworker stdout | 2025-11-04 09:15:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 15, 483970), 1, 2]) securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 15, 483970), 1, 2]) securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:21.530064+00:00 (in 5.994502 seconds) notificationworker stdout | 2025-11-04 09:15:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:15 GMT)" (scheduled at 2025-11-04 09:15:15.535095+00:00) notificationworker stdout | 2025-11-04 09:15:15,536 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:15:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:15:15,697 [285] [DEBUG] [app] Starting request: urn:request:caf81501-f057-4976-8b33-9cfc1f0ba742 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:15,701 [292] [DEBUG] [app] Starting request: urn:request:fa861931-85f8-4c8e-a86e-93d878343dfc (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:15,701 [292] [DEBUG] [app] Ending request: urn:request:fa861931-85f8-4c8e-a86e-93d878343dfc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fa861931-85f8-4c8e-a86e-93d878343dfc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:15:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:15,705 [286] [DEBUG] [app] Starting request: urn:request:c4010e17-e485-485e-a613-5c59bed380bb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:15,706 [286] [DEBUG] [app] Ending request: urn:request:c4010e17-e485-485e-a613-5c59bed380bb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c4010e17-e485-485e-a613-5c59bed380bb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:15,706 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:15,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:15,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:15,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:15,730 [285] [DEBUG] [app] Ending request: urn:request:caf81501-f057-4976-8b33-9cfc1f0ba742 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:caf81501-f057-4976-8b33-9cfc1f0ba742', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:15,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:15,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:15:15,783 [285] [DEBUG] [app] Starting request: urn:request:6deba12b-7ea4-4360-aef1-e2180fa37323 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:15,785 [292] [DEBUG] [app] Starting request: urn:request:e13e7bee-3ca6-4726-b936-e499d82bfe6b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:15,785 [292] [DEBUG] [app] Ending request: urn:request:e13e7bee-3ca6-4726-b936-e499d82bfe6b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e13e7bee-3ca6-4726-b936-e499d82bfe6b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:15:15,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:15,789 [287] [DEBUG] [app] Starting request: urn:request:e4f16456-d432-4c5b-8a1b-111412911bf8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:15,790 [287] [DEBUG] [app] Ending request: urn:request:e4f16456-d432-4c5b-8a1b-111412911bf8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e4f16456-d432-4c5b-8a1b-111412911bf8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:15,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:15,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:15,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:15,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:15,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:15,814 [285] [DEBUG] [app] Ending request: urn:request:6deba12b-7ea4-4360-aef1-e2180fa37323 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6deba12b-7ea4-4360-aef1-e2180fa37323', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:15,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:15,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.032) proxycacheblobworker stdout | 2025-11-04 09:15:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:27.122849+00:00 (in 9.999630 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:27 GMT)" (scheduled at 2025-11-04 09:15:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 17, 123484), True, datetime.datetime(2025, 11, 4, 9, 15, 17, 123484), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:15:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:45.479769+00:00 (in 28.000211 seconds) securityworker stdout | 2025-11-04 09:15:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:47 GMT)" (scheduled at 2025-11-04 09:15:17.479080+00:00) securityworker stdout | 2025-11-04 09:15:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:15:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:15:17,858 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:18,517 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:15:19,834 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:15:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:15:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:51.226294+00:00 (in 29.999499 seconds) autopruneworker stdout | 2025-11-04 09:15:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:51 GMT)" (scheduled at 2025-11-04 09:15:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:15:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247661234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:15:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:15:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:15:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:15:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:15:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:32.435813+00:00 (in 11.000973 seconds) namespacegcworker stdout | 2025-11-04 09:15:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:21 GMT)" (scheduled at 2025-11-04 09:15:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:15:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:15:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 21, 435158), True, datetime.datetime(2025, 11, 4, 9, 15, 21, 435158), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:15:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:15:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:15:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:31.530064+00:00 (in 9.999617 seconds) notificationworker stdout | 2025-11-04 09:15:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:31 GMT)" (scheduled at 2025-11-04 09:15:21.530064+00:00) notificationworker stdout | 2025-11-04 09:15:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 21, 530710), True, datetime.datetime(2025, 11, 4, 9, 15, 21, 530710), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:21,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:15:22,091 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:24,959 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:25,110 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:15:25,124 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:15:25,124 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:15:25,134 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:37.122849+00:00 (in 9.999643 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:37 GMT)" (scheduled at 2025-11-04 09:15:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 27, 123464), True, datetime.datetime(2025, 11, 4, 9, 15, 27, 123464), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:37 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:15:27,187 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:15:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:15:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:57.222846+00:00 (in 29.999633 seconds) buildlogsarchiver stdout | 2025-11-04 09:15:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:57 GMT)" (scheduled at 2025-11-04 09:15:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:15:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 15, 27, 223399), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:15:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:15:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:15:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:15:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:15:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:15:27,899 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:15:28,553 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:15:28,936 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:29,273 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:15:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:15:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:44.351216+00:00 (in 14.999551 seconds) quotatotalworker stdout | 2025-11-04 09:15:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:44 GMT)" (scheduled at 2025-11-04 09:15:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:15:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:15:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:15:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:15:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:15:29,477 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:29,623 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:29,643 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:29,718 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:30,697 [285] [DEBUG] [app] Starting request: urn:request:214370f9-d836-41b4-9ed7-a284c318a148 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:30,701 [298] [DEBUG] [app] Starting request: urn:request:cccebf62-891e-4262-b76a-2773dce757ec (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:30,702 [298] [DEBUG] [app] Ending request: urn:request:cccebf62-891e-4262-b76a-2773dce757ec (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cccebf62-891e-4262-b76a-2773dce757ec', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:15:30,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:30,706 [288] [DEBUG] [app] Starting request: urn:request:e50d8389-8a16-4b1d-b815-0578aed74af6 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:30,706 [288] [DEBUG] [app] Ending request: urn:request:e50d8389-8a16-4b1d-b815-0578aed74af6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e50d8389-8a16-4b1d-b815-0578aed74af6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:15:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:30,730 [285] [DEBUG] [app] Ending request: urn:request:214370f9-d836-41b4-9ed7-a284c318a148 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:214370f9-d836-41b4-9ed7-a284c318a148', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:15:30,783 [285] [DEBUG] [app] Starting request: urn:request:3064da16-90a5-4c8f-837c-6fc1d9572eb0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:30,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:30,785 [299] [DEBUG] [app] Starting request: urn:request:332c852f-75cb-42bd-85df-d3c0064cb705 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:30,785 [299] [DEBUG] [app] Ending request: urn:request:332c852f-75cb-42bd-85df-d3c0064cb705 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:332c852f-75cb-42bd-85df-d3c0064cb705', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:30,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:15:30,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:30,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:30,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:30,789 [288] [DEBUG] [app] Starting request: urn:request:ce10f79c-b19b-4f58-a158-b05067eb7a97 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:30,790 [288] [DEBUG] [app] Ending request: urn:request:ce10f79c-b19b-4f58-a158-b05067eb7a97 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ce10f79c-b19b-4f58-a158-b05067eb7a97', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:30,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:30,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:30,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:30,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:30,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:30,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:30,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:30,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:30,814 [285] [DEBUG] [app] Ending request: urn:request:3064da16-90a5-4c8f-837c-6fc1d9572eb0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3064da16-90a5-4c8f-837c-6fc1d9572eb0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:30,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:30,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:31,192 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:41.530064+00:00 (in 9.999617 seconds) notificationworker stdout | 2025-11-04 09:15:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:41 GMT)" (scheduled at 2025-11-04 09:15:31.530064+00:00) notificationworker stdout | 2025-11-04 09:15:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 31, 530720), True, datetime.datetime(2025, 11, 4, 9, 15, 31, 530720), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:15:31,987 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:15:32,175 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:15:32,176 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:15:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:15:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:21.434460+00:00 (in 48.998258 seconds) namespacegcworker stdout | 2025-11-04 09:15:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:32 GMT)" (scheduled at 2025-11-04 09:15:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:15:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:15:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:15:33,469 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:15:33,469 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:15:33,572 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:15:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:15:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:48.625192+00:00 (in 15.001785 seconds) exportactionlogsworker stdout | 2025-11-04 09:15:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:33 GMT)" (scheduled at 2025-11-04 09:15:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:15:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:15:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 33, 623659), True, datetime.datetime(2025, 11, 4, 9, 15, 33, 623659), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:15:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:15:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:15:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:33 GMT)" executed successfully builder stdout | 2025-11-04 09:15:34,355 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:35,134 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:15:35,148 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:15:35,148 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:15:35,158 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:38.437104+00:00 (in 3.000866 seconds) gcworker stdout | 2025-11-04 09:15:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:05 GMT)" (scheduled at 2025-11-04 09:15:35.435809+00:00) gcworker stdout | 2025-11-04 09:15:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038135436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:15:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:15:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:15:35,659 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:15:36,028 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:15:36,648 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:15:37,038 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:40.124445+00:00 (in 3.001240 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:47 GMT)" (scheduled at 2025-11-04 09:15:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 37, 123464), True, datetime.datetime(2025, 11, 4, 9, 15, 37, 123464), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:15:37,949 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:37,950 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:37,981 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:38,004 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:38,016 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:38,030 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:38,053 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:38,180 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:38,198 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:38,408 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:38,426 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:38,428 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:15:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:05.435809+00:00 (in 26.998332 seconds) gcworker stdout | 2025-11-04 09:15:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:08 GMT)" (scheduled at 2025-11-04 09:15:38.437104+00:00) gcworker stdout | 2025-11-04 09:15:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:15:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247678449, None, 1, 0]) gcworker stdout | 2025-11-04 09:15:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:15:39,207 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:15:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:15:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:01.922827+00:00 (in 21.997737 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:15:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:39 GMT)" (scheduled at 2025-11-04 09:15:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:15:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:15:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:15:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:47.122849+00:00 (in 6.998003 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:40 GMT)" (scheduled at 2025-11-04 09:15:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:15:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:15:40,976 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:51.530064+00:00 (in 9.999534 seconds) notificationworker stdout | 2025-11-04 09:15:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:51 GMT)" (scheduled at 2025-11-04 09:15:41.530064+00:00) notificationworker stdout | 2025-11-04 09:15:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 41, 530808), True, datetime.datetime(2025, 11, 4, 9, 15, 41, 530808), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:15:42,421 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:15:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:15:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:57.523915+00:00 (in 14.090659 seconds) repositorygcworker stdout | 2025-11-04 09:15:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:43 GMT)" (scheduled at 2025-11-04 09:15:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:15:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:15:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 43, 433530), True, datetime.datetime(2025, 11, 4, 9, 15, 43, 433530), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:15:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:15:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:15:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:15:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:15:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:59.351216+00:00 (in 14.999587 seconds) quotatotalworker stdout | 2025-11-04 09:15:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:59 GMT)" (scheduled at 2025-11-04 09:15:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:15:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:15:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:15:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:15:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:15:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:15:44,923 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:45,158 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:15:45,171 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:15:45,171 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:15:45,182 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:47.479080+00:00 (in 1.998883 seconds) securityworker stdout | 2025-11-04 09:15:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:15 GMT)" (scheduled at 2025-11-04 09:15:45.479769+00:00) securityworker stdout | 2025-11-04 09:15:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 45, 483595), 1, 2]) securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 45, 483595), 1, 2]) securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:15:45,697 [285] [DEBUG] [app] Starting request: urn:request:d137d5fc-7582-4190-910a-2760d6595628 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:45,701 [298] [DEBUG] [app] Starting request: urn:request:859b6029-1027-4b83-807d-652a8d41503a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:45,701 [298] [DEBUG] [app] Ending request: urn:request:859b6029-1027-4b83-807d-652a8d41503a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:859b6029-1027-4b83-807d-652a8d41503a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:45,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:15:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:45,706 [287] [DEBUG] [app] Starting request: urn:request:0a764cf9-8f64-4c6c-835a-2b87c230ac67 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:45,706 [287] [DEBUG] [app] Ending request: urn:request:0a764cf9-8f64-4c6c-835a-2b87c230ac67 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0a764cf9-8f64-4c6c-835a-2b87c230ac67', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:45,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:15:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:45,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:45,731 [285] [DEBUG] [app] Ending request: urn:request:d137d5fc-7582-4190-910a-2760d6595628 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d137d5fc-7582-4190-910a-2760d6595628', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:45,731 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:15:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:15:45,784 [288] [DEBUG] [app] Starting request: urn:request:c928e2b1-1f03-40c6-bb36-800a5d711f14 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:15:45,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:45,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:15:45,789 [299] [DEBUG] [app] Starting request: urn:request:b980bc4d-33e7-4e2b-a6f9-652a0edded23 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:15:45,789 [299] [DEBUG] [app] Ending request: urn:request:b980bc4d-33e7-4e2b-a6f9-652a0edded23 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b980bc4d-33e7-4e2b-a6f9-652a0edded23', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:15:45,790 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:45,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:45,791 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:45,793 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:15:45,793 [285] [DEBUG] [app] Starting request: urn:request:ca39bd41-565e-4187-8edb-ad23b0e84c0c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:15:45,794 [285] [DEBUG] [app] Ending request: urn:request:ca39bd41-565e-4187-8edb-ad23b0e84c0c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ca39bd41-565e-4187-8edb-ad23b0e84c0c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:15:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:45,794 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:15:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:45,794 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:45,794 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:45,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:45,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:45,802 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:45,802 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:45,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:45,817 [288] [DEBUG] [app] Ending request: urn:request:c928e2b1-1f03-40c6-bb36-800a5d711f14 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c928e2b1-1f03-40c6-bb36-800a5d711f14', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:45,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:45,818 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:15:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:15:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:15:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:57.122849+00:00 (in 9.999584 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:57 GMT)" (scheduled at 2025-11-04 09:15:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 47, 123563), True, datetime.datetime(2025, 11, 4, 9, 15, 47, 123563), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:15:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:15.479769+00:00 (in 28.000310 seconds) securityworker stdout | 2025-11-04 09:15:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:17 GMT)" (scheduled at 2025-11-04 09:15:47.479080+00:00) securityworker stdout | 2025-11-04 09:15:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:15:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 47, 482891), 1, 2]) securityworker stdout | 2025-11-04 09:15:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 47, 482891), 1, 2]) securityworker stdout | 2025-11-04 09:15:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:15:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:15:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:15:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:15:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:15:47,867 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:48,525 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:15:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:15:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:33.622985+00:00 (in 44.997356 seconds) exportactionlogsworker stdout | 2025-11-04 09:15:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:48 GMT)" (scheduled at 2025-11-04 09:15:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:15:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:15:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:15:49,843 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:15:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:15:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:21.226294+00:00 (in 29.999570 seconds) autopruneworker stdout | 2025-11-04 09:15:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:21 GMT)" (scheduled at 2025-11-04 09:15:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:15:51,237 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247691236, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:15:51,241 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:15:51,241 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:15:51,242 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:01.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:15:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:01 GMT)" (scheduled at 2025-11-04 09:15:51.530064+00:00) notificationworker stdout | 2025-11-04 09:15:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 51, 530716), True, datetime.datetime(2025, 11, 4, 9, 15, 51, 530716), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:51,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:15:52,100 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:54,968 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:55,182 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:15:55,195 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:15:55,195 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:15:55,205 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:07.122849+00:00 (in 9.999623 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:07 GMT)" (scheduled at 2025-11-04 09:15:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 57, 123513), True, datetime.datetime(2025, 11, 4, 9, 15, 57, 123513), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:15:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:09.133645+00:00 (in 12.001486 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:57 GMT)" (scheduled at 2025-11-04 09:15:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:15:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:15:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:15:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:15:57,196 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:15:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:15:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:27.222846+00:00 (in 29.999582 seconds) buildlogsarchiver stdout | 2025-11-04 09:15:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:27 GMT)" (scheduled at 2025-11-04 09:15:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:15:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 15, 57, 223471), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:15:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:15:57,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:15:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:15:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:15:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:15:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:15:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:43.432867+00:00 (in 45.908571 seconds) repositorygcworker stdout | 2025-11-04 09:15:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:57 GMT)" (scheduled at 2025-11-04 09:15:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:15:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:15:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:15:57,908 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:15:58,562 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:15:58,944 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:59,283 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:15:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:15:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:14.351216+00:00 (in 14.999522 seconds) quotatotalworker stdout | 2025-11-04 09:15:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:14 GMT)" (scheduled at 2025-11-04 09:15:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:15:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:15:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:15:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:15:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:15:59,495 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:59,635 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:59,656 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:59,739 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:00,697 [288] [DEBUG] [app] Starting request: urn:request:38de32fb-0577-4c4a-ba52-632fb1445f0b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:00,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:00,702 [298] [DEBUG] [app] Starting request: urn:request:2d3e5a90-6bd1-43c8-8ec0-62249fadb592 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:00,703 [298] [DEBUG] [app] Ending request: urn:request:2d3e5a90-6bd1-43c8-8ec0-62249fadb592 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2d3e5a90-6bd1-43c8-8ec0-62249fadb592', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:00,703 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:16:00,703 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:00,704 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:00,706 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:00,707 [285] [DEBUG] [app] Starting request: urn:request:bdb5cd4b-1914-4315-87cf-c129b0df7a3e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:00,707 [285] [DEBUG] [app] Ending request: urn:request:bdb5cd4b-1914-4315-87cf-c129b0df7a3e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bdb5cd4b-1914-4315-87cf-c129b0df7a3e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:00,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:00,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:00,708 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:00,708 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:00,708 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:00,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:00,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:00,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:00,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:00,731 [288] [DEBUG] [app] Ending request: urn:request:38de32fb-0577-4c4a-ba52-632fb1445f0b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:38de32fb-0577-4c4a-ba52-632fb1445f0b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:00,731 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:16:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:16:00,784 [285] [DEBUG] [app] Starting request: urn:request:6de5c9f2-b112-484f-b062-388fb5e583b7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:00,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:00,788 [298] [DEBUG] [app] Starting request: urn:request:2a9b86ae-fd02-41aa-abd1-4be503419bb2 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:00,788 [298] [DEBUG] [app] Ending request: urn:request:2a9b86ae-fd02-41aa-abd1-4be503419bb2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2a9b86ae-fd02-41aa-abd1-4be503419bb2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:16:00,789 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:00,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:00,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:00,792 [286] [DEBUG] [app] Starting request: urn:request:d5cd83bd-2ce8-41a6-8e9d-b3596c18f80c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:00,793 [286] [DEBUG] [app] Ending request: urn:request:d5cd83bd-2ce8-41a6-8e9d-b3596c18f80c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d5cd83bd-2ce8-41a6-8e9d-b3596c18f80c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:00,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:00,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:00,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:00,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:00,817 [285] [DEBUG] [app] Ending request: urn:request:6de5c9f2-b112-484f-b062-388fb5e583b7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6de5c9f2-b112-484f-b062-388fb5e583b7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:00,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:01,201 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:11.530064+00:00 (in 9.999545 seconds) notificationworker stdout | 2025-11-04 09:16:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:11 GMT)" (scheduled at 2025-11-04 09:16:01.530064+00:00) notificationworker stdout | 2025-11-04 09:16:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 1, 530841), True, datetime.datetime(2025, 11, 4, 9, 16, 1, 530841), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:01,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:16:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:16:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:39.924652+00:00 (in 38.001366 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:16:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:01 GMT)" (scheduled at 2025-11-04 09:16:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:16:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:16:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 1, 923596), True, datetime.datetime(2025, 11, 4, 9, 16, 1, 923596), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:16:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:16:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:16:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:16:01,995 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:16:02,184 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:16:02,185 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:16:03,477 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:16:03,477 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:16:03,582 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:04,365 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:05,206 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:16:05,219 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:16:05,219 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:16:05,230 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:08.437104+00:00 (in 3.000899 seconds) gcworker stdout | 2025-11-04 09:16:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:35 GMT)" (scheduled at 2025-11-04 09:16:05.435809+00:00) gcworker stdout | 2025-11-04 09:16:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038165436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:16:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:16:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:16:05,668 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:16:06,037 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:16:06,658 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:16:07,047 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:17.122849+00:00 (in 9.999618 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:17 GMT)" (scheduled at 2025-11-04 09:16:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 7, 123478), True, datetime.datetime(2025, 11, 4, 9, 16, 7, 123478), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:16:07,961 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:07,962 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:07,994 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:08,017 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:08,026 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:08,041 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:08,067 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:08,201 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:08,209 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:08,419 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:16:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:35.435809+00:00 (in 26.998293 seconds) gunicorn-secscan stdout | 2025-11-04 09:16:08,436 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:16:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:38 GMT)" (scheduled at 2025-11-04 09:16:08.437104+00:00) gcworker stdout | 2025-11-04 09:16:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-secscan stdout | 2025-11-04 09:16:08,438 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:16:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247708449, None, 1, 0]) gcworker stdout | 2025-11-04 09:16:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:38 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:16:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:16:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:57.131795+00:00 (in 47.997736 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:09 GMT)" (scheduled at 2025-11-04 09:16:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:16:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:16:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:16:09,225 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:16:10,984 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:15.535095+00:00 (in 4.004565 seconds) notificationworker stdout | 2025-11-04 09:16:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:21 GMT)" (scheduled at 2025-11-04 09:16:11.530064+00:00) notificationworker stdout | 2025-11-04 09:16:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 11, 530820), True, datetime.datetime(2025, 11, 4, 9, 16, 11, 530820), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:16:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:16:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:12.222828+00:00 (in 59.999565 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:16:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:12 GMT)" (scheduled at 2025-11-04 09:16:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:16:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:16:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:16:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:16:12,429 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:16:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:16:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:29.351216+00:00 (in 14.999589 seconds) quotatotalworker stdout | 2025-11-04 09:16:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:29 GMT)" (scheduled at 2025-11-04 09:16:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:16:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:16:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:16:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:16:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:16:14,931 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:15,230 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:16:15,243 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:16:15,243 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:16:15,253 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:17.479080+00:00 (in 1.998887 seconds) securityworker stdout | 2025-11-04 09:16:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:45 GMT)" (scheduled at 2025-11-04 09:16:15.479769+00:00) securityworker stdout | 2025-11-04 09:16:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 15, 483930), 1, 2]) securityworker stdout | 2025-11-04 09:16:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 15, 483930), 1, 2]) securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:21.530064+00:00 (in 5.994521 seconds) notificationworker stdout | 2025-11-04 09:16:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:15 GMT)" (scheduled at 2025-11-04 09:16:15.535095+00:00) notificationworker stdout | 2025-11-04 09:16:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:16:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:16:15,697 [285] [DEBUG] [app] Starting request: urn:request:15b595df-6f8a-4854-9a30-bc61bbdd2544 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:15,701 [299] [DEBUG] [app] Starting request: urn:request:176bb954-8982-460f-b87c-b353be8fe10f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:15,701 [299] [DEBUG] [app] Ending request: urn:request:176bb954-8982-460f-b87c-b353be8fe10f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:176bb954-8982-460f-b87c-b353be8fe10f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:16:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:15,706 [288] [DEBUG] [app] Starting request: urn:request:2a6ef2ef-d12b-429d-b64d-5fd668528fd4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:15,706 [288] [DEBUG] [app] Ending request: urn:request:2a6ef2ef-d12b-429d-b64d-5fd668528fd4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2a6ef2ef-d12b-429d-b64d-5fd668528fd4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:16:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:15,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:15,730 [285] [DEBUG] [app] Ending request: urn:request:15b595df-6f8a-4854-9a30-bc61bbdd2544 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:15b595df-6f8a-4854-9a30-bc61bbdd2544', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:15,730 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:16:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:16:15,783 [285] [DEBUG] [app] Starting request: urn:request:0086770f-8c50-4f47-8276-4e06f367f01c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:15,785 [299] [DEBUG] [app] Starting request: urn:request:f41f9347-8d16-400f-9a01-3535453288c4 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:15,785 [299] [DEBUG] [app] Ending request: urn:request:f41f9347-8d16-400f-9a01-3535453288c4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f41f9347-8d16-400f-9a01-3535453288c4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:15,785 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:16:15,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:15,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:15,789 [287] [DEBUG] [app] Starting request: urn:request:4cfa0270-6b71-4b1a-98ab-ae55bdb8120f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:15,789 [287] [DEBUG] [app] Ending request: urn:request:4cfa0270-6b71-4b1a-98ab-ae55bdb8120f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4cfa0270-6b71-4b1a-98ab-ae55bdb8120f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:15,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:15,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:15,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:15,813 [285] [DEBUG] [app] Ending request: urn:request:0086770f-8c50-4f47-8276-4e06f367f01c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0086770f-8c50-4f47-8276-4e06f367f01c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:15,813 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) gunicorn-web stdout | 2025-11-04 09:16:15,813 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:16:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:27.122849+00:00 (in 9.999580 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:27 GMT)" (scheduled at 2025-11-04 09:16:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 17, 123578), True, datetime.datetime(2025, 11, 4, 9, 16, 17, 123578), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:16:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:45.479769+00:00 (in 28.000228 seconds) securityworker stdout | 2025-11-04 09:16:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:47 GMT)" (scheduled at 2025-11-04 09:16:17.479080+00:00) securityworker stdout | 2025-11-04 09:16:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:16:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:16:17,876 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:18,534 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:16:19,853 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:16:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:16:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:51.226294+00:00 (in 29.999560 seconds) autopruneworker stdout | 2025-11-04 09:16:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:51 GMT)" (scheduled at 2025-11-04 09:16:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:16:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247721234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:16:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:16:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:16:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:16:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:16:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:32.435813+00:00 (in 11.000935 seconds) namespacegcworker stdout | 2025-11-04 09:16:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:21 GMT)" (scheduled at 2025-11-04 09:16:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:16:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:16:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 21, 435166), True, datetime.datetime(2025, 11, 4, 9, 16, 21, 435166), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:16:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:16:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:16:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:31.530064+00:00 (in 9.999588 seconds) notificationworker stdout | 2025-11-04 09:16:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:31 GMT)" (scheduled at 2025-11-04 09:16:21.530064+00:00) notificationworker stdout | 2025-11-04 09:16:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 21, 530734), True, datetime.datetime(2025, 11, 4, 9, 16, 21, 530734), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:16:22,109 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:24,976 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:25,254 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:16:25,267 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:16:25,267 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:16:25,277 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:37.122849+00:00 (in 9.999607 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:37 GMT)" (scheduled at 2025-11-04 09:16:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 27, 123548), True, datetime.datetime(2025, 11, 4, 9, 16, 27, 123548), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:37 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:16:27,205 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:16:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:16:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:57.222846+00:00 (in 29.999630 seconds) buildlogsarchiver stdout | 2025-11-04 09:16:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:57 GMT)" (scheduled at 2025-11-04 09:16:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:16:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 16, 27, 223402), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:16:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:16:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:16:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:16:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:16:27,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:16:27,917 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:16:28,571 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:16:28,953 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:29,294 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:16:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:16:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:44.351216+00:00 (in 14.999511 seconds) quotatotalworker stdout | 2025-11-04 09:16:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:44 GMT)" (scheduled at 2025-11-04 09:16:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:16:29,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:16:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:16:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:16:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:16:29,506 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:29,649 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:29,669 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:29,754 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:30,697 [286] [DEBUG] [app] Starting request: urn:request:c87080e0-2650-4b38-864f-e9159279b0d6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:30,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:30,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:30,701 [292] [DEBUG] [app] Starting request: urn:request:338566bf-c7ed-4daf-8cc8-d36593b800c2 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:30,702 [292] [DEBUG] [app] Ending request: urn:request:338566bf-c7ed-4daf-8cc8-d36593b800c2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:338566bf-c7ed-4daf-8cc8-d36593b800c2', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:16:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:30,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:30,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:30,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:30,706 [285] [DEBUG] [app] Starting request: urn:request:5feebb92-6013-42ac-bb54-5238404b7309 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:30,706 [285] [DEBUG] [app] Ending request: urn:request:5feebb92-6013-42ac-bb54-5238404b7309 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5feebb92-6013-42ac-bb54-5238404b7309', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:30,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:30,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:30,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:30,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:30,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:30,715 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:30,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:30,728 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:30,731 [286] [DEBUG] [app] Ending request: urn:request:c87080e0-2650-4b38-864f-e9159279b0d6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c87080e0-2650-4b38-864f-e9159279b0d6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:30,731 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:30,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:16:30,784 [288] [DEBUG] [app] Starting request: urn:request:213b544f-7ee5-446e-9f86-af8b5801e4ae (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:30,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:30,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:30,788 [293] [DEBUG] [app] Starting request: urn:request:39cb8831-6692-48e0-a28f-0c5ef6d61d94 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:30,789 [293] [DEBUG] [app] Ending request: urn:request:39cb8831-6692-48e0-a28f-0c5ef6d61d94 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:39cb8831-6692-48e0-a28f-0c5ef6d61d94', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:16:30,789 [293] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:30,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:30,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:30,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:30,792 [285] [DEBUG] [app] Starting request: urn:request:c23acb60-face-4b3a-91b7-8b206f694329 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:30,793 [285] [DEBUG] [app] Ending request: urn:request:c23acb60-face-4b3a-91b7-8b206f694329 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c23acb60-face-4b3a-91b7-8b206f694329', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:30,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:16:30,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:30,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:30,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:30,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:30,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:30,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:30,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:30,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:30,817 [288] [DEBUG] [app] Ending request: urn:request:213b544f-7ee5-446e-9f86-af8b5801e4ae (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:213b544f-7ee5-446e-9f86-af8b5801e4ae', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:30,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:30,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:31,210 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:41.530064+00:00 (in 9.999593 seconds) notificationworker stdout | 2025-11-04 09:16:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:41 GMT)" (scheduled at 2025-11-04 09:16:31.530064+00:00) notificationworker stdout | 2025-11-04 09:16:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 31, 530742), True, datetime.datetime(2025, 11, 4, 9, 16, 31, 530742), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:16:32,004 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:16:32,193 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:16:32,193 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:16:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:16:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:21.434460+00:00 (in 48.998211 seconds) namespacegcworker stdout | 2025-11-04 09:16:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:32 GMT)" (scheduled at 2025-11-04 09:16:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:16:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:16:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:32 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:16:33,486 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:16:33,486 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:16:33,590 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:16:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:16:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:48.625192+00:00 (in 15.001840 seconds) exportactionlogsworker stdout | 2025-11-04 09:16:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:33 GMT)" (scheduled at 2025-11-04 09:16:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:16:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:16:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 33, 623615), True, datetime.datetime(2025, 11, 4, 9, 16, 33, 623615), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:16:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:16:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:16:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:33 GMT)" executed successfully builder stdout | 2025-11-04 09:16:34,374 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:35,278 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:16:35,291 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:16:35,291 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:16:35,301 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:38.437104+00:00 (in 3.000897 seconds) gcworker stdout | 2025-11-04 09:16:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:05 GMT)" (scheduled at 2025-11-04 09:16:35.435809+00:00) gcworker stdout | 2025-11-04 09:16:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038195436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:16:35,449 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:16:35,449 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:16:35,677 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:16:36,046 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:16:36,668 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:16:37,056 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:40.124445+00:00 (in 3.001179 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:47 GMT)" (scheduled at 2025-11-04 09:16:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 37, 123521), True, datetime.datetime(2025, 11, 4, 9, 16, 37, 123521), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:16:37,977 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:37,979 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:38,006 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:38,028 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:38,041 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:38,052 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:38,080 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:38,216 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:38,224 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:38,429 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:16:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:05.435809+00:00 (in 26.998233 seconds) gcworker stdout | 2025-11-04 09:16:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:08 GMT)" (scheduled at 2025-11-04 09:16:38.437104+00:00) gcworker stdout | 2025-11-04 09:16:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-secscan stdout | 2025-11-04 09:16:38,446 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:38,447 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:16:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247738449, None, 1, 0]) gcworker stdout | 2025-11-04 09:16:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:08 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:16:39,234 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:16:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:16:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:01.922827+00:00 (in 21.997730 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:16:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:39 GMT)" (scheduled at 2025-11-04 09:16:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:16:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:16:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:16:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:47.122849+00:00 (in 6.997998 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:40 GMT)" (scheduled at 2025-11-04 09:16:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:16:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:16:40,993 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:51.530064+00:00 (in 9.999515 seconds) notificationworker stdout | 2025-11-04 09:16:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:51 GMT)" (scheduled at 2025-11-04 09:16:41.530064+00:00) notificationworker stdout | 2025-11-04 09:16:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 41, 530825), True, datetime.datetime(2025, 11, 4, 9, 16, 41, 530825), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:16:42,438 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:16:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:16:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:57.523915+00:00 (in 14.090646 seconds) repositorygcworker stdout | 2025-11-04 09:16:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:43 GMT)" (scheduled at 2025-11-04 09:16:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:16:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:16:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 43, 433603), True, datetime.datetime(2025, 11, 4, 9, 16, 43, 433603), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:16:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:16:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:16:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:16:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:16:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:59.351216+00:00 (in 14.999551 seconds) quotatotalworker stdout | 2025-11-04 09:16:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:59 GMT)" (scheduled at 2025-11-04 09:16:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:16:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:16:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:16:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:16:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:16:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:16:44,940 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:45,301 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:16:45,315 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:16:45,315 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:16:45,325 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:47.479080+00:00 (in 1.998883 seconds) securityworker stdout | 2025-11-04 09:16:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:15 GMT)" (scheduled at 2025-11-04 09:16:45.479769+00:00) securityworker stdout | 2025-11-04 09:16:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 45, 484067), 1, 2]) securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 45, 484067), 1, 2]) securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:16:45,696 [285] [DEBUG] [app] Starting request: urn:request:11945e3e-1bee-41f2-b168-7533033c5e06 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:45,701 [298] [DEBUG] [app] Starting request: urn:request:66896c6c-bc1a-4f2f-b7c5-b98163efd9ef (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:45,701 [298] [DEBUG] [app] Ending request: urn:request:66896c6c-bc1a-4f2f-b7c5-b98163efd9ef (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:66896c6c-bc1a-4f2f-b7c5-b98163efd9ef', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:45,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:16:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:45,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:45,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:45,705 [288] [DEBUG] [app] Starting request: urn:request:ff3e5134-d4e2-4cf8-b10a-54757520fd78 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:45,706 [288] [DEBUG] [app] Ending request: urn:request:ff3e5134-d4e2-4cf8-b10a-54757520fd78 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ff3e5134-d4e2-4cf8-b10a-54757520fd78', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:45,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:45,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:45,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:45,730 [285] [DEBUG] [app] Ending request: urn:request:11945e3e-1bee-41f2-b168-7533033c5e06 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:11945e3e-1bee-41f2-b168-7533033c5e06', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:45,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:16:45,784 [288] [DEBUG] [app] Starting request: urn:request:f1cf4ee3-daa2-4798-bf72-7891c189fda0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:16:45,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:45,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:16:45,788 [292] [DEBUG] [app] Starting request: urn:request:c794c798-fa75-4eac-8033-6a713b064ae5 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:16:45,788 [292] [DEBUG] [app] Ending request: urn:request:c794c798-fa75-4eac-8033-6a713b064ae5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c794c798-fa75-4eac-8033-6a713b064ae5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:16:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:45,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:45,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:45,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:16:45,792 [287] [DEBUG] [app] Starting request: urn:request:d139432d-9c5f-41c7-9bd7-cb98a0f75c42 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:16:45,793 [287] [DEBUG] [app] Ending request: urn:request:d139432d-9c5f-41c7-9bd7-cb98a0f75c42 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d139432d-9c5f-41c7-9bd7-cb98a0f75c42', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:45,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:16:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:16:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:45,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:45,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:45,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:45,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:45,817 [288] [DEBUG] [app] Ending request: urn:request:f1cf4ee3-daa2-4798-bf72-7891c189fda0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f1cf4ee3-daa2-4798-bf72-7891c189fda0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:45,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:16:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:16:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:16:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:57.122849+00:00 (in 9.999611 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:57 GMT)" (scheduled at 2025-11-04 09:16:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 47, 123484), True, datetime.datetime(2025, 11, 4, 9, 16, 47, 123484), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:16:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:15.479769+00:00 (in 28.000264 seconds) securityworker stdout | 2025-11-04 09:16:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:17 GMT)" (scheduled at 2025-11-04 09:16:47.479080+00:00) securityworker stdout | 2025-11-04 09:16:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:47,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:16:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:47,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:47,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:47,504 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:47,504 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:47,504 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:47,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:47,505 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 47, 484699), 1, 2]) securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:16:47,508 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:47,509 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 47, 484699), 1, 2]) securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:47,512 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:16:47,512 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:16:47,512 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:47,513 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:16:47,513 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:47,513 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:16:47,513 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:16:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:47,513 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:47,513 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:16:47,885 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:48,543 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:16:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:16:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:58.624878+00:00 (in 9.999250 seconds) exportactionlogsworker stdout | 2025-11-04 09:16:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:48 GMT)" (scheduled at 2025-11-04 09:16:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:16:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:16:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:16:49,862 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:16:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:16:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:21.226294+00:00 (in 29.999539 seconds) autopruneworker stdout | 2025-11-04 09:16:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:21 GMT)" (scheduled at 2025-11-04 09:16:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:16:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247751235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:16:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:16:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:16:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:01.530064+00:00 (in 9.999515 seconds) notificationworker stdout | 2025-11-04 09:16:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:01 GMT)" (scheduled at 2025-11-04 09:16:51.530064+00:00) notificationworker stdout | 2025-11-04 09:16:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 51, 530819), True, datetime.datetime(2025, 11, 4, 9, 16, 51, 530819), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:51,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:16:52,118 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:54,985 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:55,325 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:16:55,339 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:16:55,339 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:16:55,349 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:07.122849+00:00 (in 9.999616 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:07 GMT)" (scheduled at 2025-11-04 09:16:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 57, 123536), True, datetime.datetime(2025, 11, 4, 9, 16, 57, 123536), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:16:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:09.133645+00:00 (in 12.001452 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:57 GMT)" (scheduled at 2025-11-04 09:16:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:16:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:16:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:16:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:16:57,214 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:16:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:16:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:27.222846+00:00 (in 29.999607 seconds) buildlogsarchiver stdout | 2025-11-04 09:16:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:27 GMT)" (scheduled at 2025-11-04 09:16:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:16:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 16, 57, 223463), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:16:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:16:57,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:16:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:16:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:16:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:16:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:16:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:43.432867+00:00 (in 45.908567 seconds) repositorygcworker stdout | 2025-11-04 09:16:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:57 GMT)" (scheduled at 2025-11-04 09:16:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:16:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:16:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:16:57,926 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:16:58,579 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:16:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:16:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:33.622985+00:00 (in 34.997712 seconds) exportactionlogsworker stdout | 2025-11-04 09:16:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:21:58 GMT)" (scheduled at 2025-11-04 09:16:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:16:58,625 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 16, 58, 625514), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:16:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 16, 58, 625514), True, datetime.datetime(2025, 11, 4, 9, 16, 58, 625514), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:16:58,641 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 16, 58, 625514), True, datetime.datetime(2025, 11, 4, 9, 16, 58, 625514), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 16, 58, 625514), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:16:58,644 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:16:58,644 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:21:58 GMT)" executed successfully teamsyncworker stdout | 2025-11-04 09:16:58,961 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:59,304 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:16:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:16:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:14.351216+00:00 (in 14.999540 seconds) quotatotalworker stdout | 2025-11-04 09:16:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:14 GMT)" (scheduled at 2025-11-04 09:16:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:16:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:16:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:16:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:16:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:16:59,519 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:59,662 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:59,682 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:59,769 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:00,697 [288] [DEBUG] [app] Starting request: urn:request:80cc9075-1d5f-46ed-8a39-f4444e17cb4f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:00,701 [299] [DEBUG] [app] Starting request: urn:request:2a2cc90b-7dcf-4504-ad00-9c33d157e98e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:00,702 [299] [DEBUG] [app] Ending request: urn:request:2a2cc90b-7dcf-4504-ad00-9c33d157e98e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2a2cc90b-7dcf-4504-ad00-9c33d157e98e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:17:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:00,706 [285] [DEBUG] [app] Starting request: urn:request:bde1632d-c21b-452f-94b7-b3c6ae94117f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:00,706 [285] [DEBUG] [app] Ending request: urn:request:bde1632d-c21b-452f-94b7-b3c6ae94117f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bde1632d-c21b-452f-94b7-b3c6ae94117f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:00,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:00,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:00,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:00,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:00,730 [288] [DEBUG] [app] Ending request: urn:request:80cc9075-1d5f-46ed-8a39-f4444e17cb4f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:80cc9075-1d5f-46ed-8a39-f4444e17cb4f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:00,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:17:00,784 [287] [DEBUG] [app] Starting request: urn:request:160554e6-8584-49f4-a18e-16d334c75df3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:00,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:00,788 [295] [DEBUG] [app] Starting request: urn:request:aab67cbd-0831-4bc3-b631-6c821b8eda22 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:00,789 [295] [DEBUG] [app] Ending request: urn:request:aab67cbd-0831-4bc3-b631-6c821b8eda22 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:aab67cbd-0831-4bc3-b631-6c821b8eda22', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:17:00,789 [295] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:00,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:00,790 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:00,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:00,793 [285] [DEBUG] [app] Starting request: urn:request:3bbf373b-5791-4216-a588-12d010bc17a3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:00,793 [285] [DEBUG] [app] Ending request: urn:request:3bbf373b-5791-4216-a588-12d010bc17a3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3bbf373b-5791-4216-a588-12d010bc17a3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:00,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:00,794 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:00,794 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:00,794 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:00,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:00,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:00,811 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:00,814 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:00,817 [287] [DEBUG] [app] Ending request: urn:request:160554e6-8584-49f4-a18e-16d334c75df3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:160554e6-8584-49f4-a18e-16d334c75df3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:00,817 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:00,818 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:01,219 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:11.530064+00:00 (in 9.999492 seconds) notificationworker stdout | 2025-11-04 09:17:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:11 GMT)" (scheduled at 2025-11-04 09:17:01.530064+00:00) notificationworker stdout | 2025-11-04 09:17:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 1, 530859), True, datetime.datetime(2025, 11, 4, 9, 17, 1, 530859), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:17:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:17:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:21.924304+00:00 (in 20.001066 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:17:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:01 GMT)" (scheduled at 2025-11-04 09:17:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:17:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:17:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 1, 923560), True, datetime.datetime(2025, 11, 4, 9, 17, 1, 923560), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:17:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:17:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:17:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:17:02,013 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:17:02,202 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:17:02,214 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:17:03,494 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:17:03,494 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:17:03,599 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:04,384 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:05,349 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:17:05,363 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:17:05,363 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:17:05,373 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:08.437104+00:00 (in 3.000873 seconds) gcworker stdout | 2025-11-04 09:17:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:35 GMT)" (scheduled at 2025-11-04 09:17:05.435809+00:00) gcworker stdout | 2025-11-04 09:17:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038225436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:17:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:17:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:17:05,686 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:17:06,055 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:17:06,677 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:17:07,070 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:17.122849+00:00 (in 9.999619 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:17 GMT)" (scheduled at 2025-11-04 09:17:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 7, 123473), True, datetime.datetime(2025, 11, 4, 9, 17, 7, 123473), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:17:07,992 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:07,988 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:08,031 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:08,038 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:08,062 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:08,065 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:08,091 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:08,231 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:08,238 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:17:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:35.435809+00:00 (in 26.998293 seconds) gcworker stdout | 2025-11-04 09:17:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:38 GMT)" (scheduled at 2025-11-04 09:17:08.437104+00:00) gcworker stdout | 2025-11-04 09:17:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-secscan stdout | 2025-11-04 09:17:08,440 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:17:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247768449, None, 1, 0]) gcworker stdout | 2025-11-04 09:17:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:38 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:17:08,457 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:08,457 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:17:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:17:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:57.131795+00:00 (in 47.997695 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:09 GMT)" (scheduled at 2025-11-04 09:17:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:17:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:17:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:17:09,243 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:17:11,002 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:15.535095+00:00 (in 4.004189 seconds) notificationworker stdout | 2025-11-04 09:17:11,531 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:21 GMT)" (scheduled at 2025-11-04 09:17:11.530064+00:00) notificationworker stdout | 2025-11-04 09:17:11,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 11, 531232), True, datetime.datetime(2025, 11, 4, 9, 17, 11, 531232), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:17:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:17:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:12.222828+00:00 (in 59.999564 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:17:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:12 GMT)" (scheduled at 2025-11-04 09:17:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:17:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:17:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:17:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:17:12,447 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:17:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:17:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:29.351216+00:00 (in 14.999551 seconds) quotatotalworker stdout | 2025-11-04 09:17:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:29 GMT)" (scheduled at 2025-11-04 09:17:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:17:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:17:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:17:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:17:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:17:14,949 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:15,374 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:17:15,388 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:17:15,388 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:17:15,398 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:17:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:17.479080+00:00 (in 1.998901 seconds) securityworker stdout | 2025-11-04 09:17:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:45 GMT)" (scheduled at 2025-11-04 09:17:15.479769+00:00) securityworker stdout | 2025-11-04 09:17:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:17:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:17:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 15, 483852), 1, 2]) securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 15, 483852), 1, 2]) securityworker stdout | 2025-11-04 09:17:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:17:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:45 GMT)" executed successfully securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker notificationworker stdout | 2025-11-04 09:17:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:21.530064+00:00 (in 5.994477 seconds) notificationworker stdout | 2025-11-04 09:17:15,536 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:15 GMT)" (scheduled at 2025-11-04 09:17:15.535095+00:00) notificationworker stdout | 2025-11-04 09:17:15,536 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:17:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:17:15,696 [288] [DEBUG] [app] Starting request: urn:request:ce4c7d42-5483-43a3-9cef-de6cbdc04358 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:15,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:15,701 [299] [DEBUG] [app] Starting request: urn:request:a795133a-47ce-43e5-a3ce-2fce6ae6f417 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:15,701 [299] [DEBUG] [app] Ending request: urn:request:a795133a-47ce-43e5-a3ce-2fce6ae6f417 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a795133a-47ce-43e5-a3ce-2fce6ae6f417', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:17:15,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:17:15,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:15,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:15,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:15,705 [287] [DEBUG] [app] Starting request: urn:request:d443c93f-c659-49f6-9705-c077a250af8f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:15,705 [287] [DEBUG] [app] Ending request: urn:request:d443c93f-c659-49f6-9705-c077a250af8f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d443c93f-c659-49f6-9705-c077a250af8f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:15,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:15,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:15,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:15,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:15,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:15,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:15,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:15,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:15,730 [288] [DEBUG] [app] Ending request: urn:request:ce4c7d42-5483-43a3-9cef-de6cbdc04358 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ce4c7d42-5483-43a3-9cef-de6cbdc04358', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:15,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:15,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:17:15,784 [285] [DEBUG] [app] Starting request: urn:request:1bbb1e45-94e7-4b47-9abf-9d277adbdce6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:15,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:15,788 [292] [DEBUG] [app] Starting request: urn:request:9756482c-5b84-41eb-a946-9d690388d2c0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:15,788 [292] [DEBUG] [app] Ending request: urn:request:9756482c-5b84-41eb-a946-9d690388d2c0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9756482c-5b84-41eb-a946-9d690388d2c0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:17:15,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:17:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:15,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:15,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:15,793 [288] [DEBUG] [app] Starting request: urn:request:ccfa094e-33e5-483b-a6dd-e2935d88742f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:15,793 [288] [DEBUG] [app] Ending request: urn:request:ccfa094e-33e5-483b-a6dd-e2935d88742f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ccfa094e-33e5-483b-a6dd-e2935d88742f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:15,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:15,794 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:15,794 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:15,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:15,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:15,802 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:15,802 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:15,812 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:15,815 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:15,818 [285] [DEBUG] [app] Ending request: urn:request:1bbb1e45-94e7-4b47-9abf-9d277adbdce6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1bbb1e45-94e7-4b47-9abf-9d277adbdce6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:15,818 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:15,819 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) proxycacheblobworker stdout | 2025-11-04 09:17:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:27.122849+00:00 (in 9.999644 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:27 GMT)" (scheduled at 2025-11-04 09:17:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 17, 123438), True, datetime.datetime(2025, 11, 4, 9, 17, 17, 123438), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:17:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:45.479769+00:00 (in 28.000204 seconds) securityworker stdout | 2025-11-04 09:17:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:47 GMT)" (scheduled at 2025-11-04 09:17:17.479080+00:00) securityworker stdout | 2025-11-04 09:17:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:17:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:17:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:17:17,894 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:18,552 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:17:19,871 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:17:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:17:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:51.226294+00:00 (in 29.999526 seconds) autopruneworker stdout | 2025-11-04 09:17:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:51 GMT)" (scheduled at 2025-11-04 09:17:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:17:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247781235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:17:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:17:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:17:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:17:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:17:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:32.435813+00:00 (in 11.000924 seconds) namespacegcworker stdout | 2025-11-04 09:17:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:21 GMT)" (scheduled at 2025-11-04 09:17:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:17:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:17:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 21, 435175), True, datetime.datetime(2025, 11, 4, 9, 17, 21, 435175), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:17:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:17:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:17:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:31.530064+00:00 (in 9.999582 seconds) notificationworker stdout | 2025-11-04 09:17:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:31 GMT)" (scheduled at 2025-11-04 09:17:21.530064+00:00) notificationworker stdout | 2025-11-04 09:17:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 21, 530780), True, datetime.datetime(2025, 11, 4, 9, 17, 21, 530780), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:31 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:17:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:17:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:39.924652+00:00 (in 17.999902 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:17:21,924 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:21 GMT)" (scheduled at 2025-11-04 09:17:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:17:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 17, 21, 925005), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:17:21,937 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 21, 925005), True, datetime.datetime(2025, 11, 4, 9, 17, 21, 925005), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:17:21,941 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 21, 925005), True, datetime.datetime(2025, 11, 4, 9, 17, 21, 925005), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 17, 21, 925005), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:17:21,944 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:17:21,944 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:21 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:17:22,127 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:24,994 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:25,398 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:17:25,411 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:17:25,411 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:17:25,422 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:37.122849+00:00 (in 9.999608 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:37 GMT)" (scheduled at 2025-11-04 09:17:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 27, 123513), True, datetime.datetime(2025, 11, 4, 9, 17, 27, 123513), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:17:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:17:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:57.222846+00:00 (in 29.999631 seconds) buildlogsarchiver stdout | 2025-11-04 09:17:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:57 GMT)" (scheduled at 2025-11-04 09:17:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:17:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 17, 27, 223424), False, 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:17:27,223 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:17:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:17:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:17:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:17:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:17:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:17:27,934 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:17:28,589 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:17:28,970 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:29,314 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:17:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:17:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:44.351216+00:00 (in 14.999547 seconds) quotatotalworker stdout | 2025-11-04 09:17:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:44 GMT)" (scheduled at 2025-11-04 09:17:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:17:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:17:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:17:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:17:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:17:29,530 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:29,677 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:29,696 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:29,783 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:30,697 [287] [DEBUG] [app] Starting request: urn:request:acbe5203-8970-42c2-bda9-1230bb0bd21d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:30,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:30,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:30,701 [292] [DEBUG] [app] Starting request: urn:request:5e233c7b-3e5f-425f-9278-1b93ddd150d4 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:30,701 [292] [DEBUG] [app] Ending request: urn:request:5e233c7b-3e5f-425f-9278-1b93ddd150d4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5e233c7b-3e5f-425f-9278-1b93ddd150d4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:17:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:17:30,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:30,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:30,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:30,705 [285] [DEBUG] [app] Starting request: urn:request:32913471-3d40-4f31-82e0-e886a366dcd6 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:30,706 [285] [DEBUG] [app] Ending request: urn:request:32913471-3d40-4f31-82e0-e886a366dcd6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:32913471-3d40-4f31-82e0-e886a366dcd6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:30,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:30,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:30,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:30,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:30,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:30,723 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:30,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:30,730 [287] [DEBUG] [app] Ending request: urn:request:acbe5203-8970-42c2-bda9-1230bb0bd21d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:acbe5203-8970-42c2-bda9-1230bb0bd21d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:30,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:30,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:17:30,783 [285] [DEBUG] [app] Starting request: urn:request:dea76154-0655-4b6b-9e19-b4654af854cd (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:30,787 [292] [DEBUG] [app] Starting request: urn:request:ce9a6e96-9f62-4a05-a0ac-b1a017232adc (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:30,788 [292] [DEBUG] [app] Ending request: urn:request:ce9a6e96-9f62-4a05-a0ac-b1a017232adc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ce9a6e96-9f62-4a05-a0ac-b1a017232adc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:17:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:30,791 [287] [DEBUG] [app] Starting request: urn:request:561dc513-985f-4563-be1e-6d8a308e5505 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:30,792 [287] [DEBUG] [app] Ending request: urn:request:561dc513-985f-4563-be1e-6d8a308e5505 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:561dc513-985f-4563-be1e-6d8a308e5505', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:30,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:30,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:30,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:30,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:30,792 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:30,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:30,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:30,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:30,816 [285] [DEBUG] [app] Ending request: urn:request:dea76154-0655-4b6b-9e19-b4654af854cd (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:dea76154-0655-4b6b-9e19-b4654af854cd', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:30,816 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:17:30,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:17:31,228 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:41.530064+00:00 (in 9.999485 seconds) notificationworker stdout | 2025-11-04 09:17:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:41 GMT)" (scheduled at 2025-11-04 09:17:31.530064+00:00) notificationworker stdout | 2025-11-04 09:17:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 31, 530908), True, datetime.datetime(2025, 11, 4, 9, 17, 31, 530908), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:17:32,022 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:17:32,211 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:17:32,223 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:17:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:17:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:08.435432+00:00 (in 35.999220 seconds) namespacegcworker stdout | 2025-11-04 09:17:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:32 GMT)" (scheduled at 2025-11-04 09:17:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:17:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:17:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:17:33,502 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:17:33,503 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:17:33,608 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:17:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:17:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:48.625192+00:00 (in 15.001853 seconds) exportactionlogsworker stdout | 2025-11-04 09:17:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:33 GMT)" (scheduled at 2025-11-04 09:17:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:17:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:17:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 33, 623610), True, datetime.datetime(2025, 11, 4, 9, 17, 33, 623610), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:17:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:17:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:17:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:33 GMT)" executed successfully builder stdout | 2025-11-04 09:17:34,393 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:35,422 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:17:35,435 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:17:35,435 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics gcworker stdout | 2025-11-04 09:17:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:38.437104+00:00 (in 3.000916 seconds) gcworker stdout | 2025-11-04 09:17:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:05 GMT)" (scheduled at 2025-11-04 09:17:35.435809+00:00) gcworker stdout | 2025-11-04 09:17:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038255436, 1209600, True, 3, 500, 1, 0]) builder stdout | 2025-11-04 09:17:35,445 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:35,449 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:17:35,449 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:35,449 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:17:35,694 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:17:36,064 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:17:36,687 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:17:37,080 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:40.124445+00:00 (in 3.001231 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:47 GMT)" (scheduled at 2025-11-04 09:17:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 37, 123507), True, datetime.datetime(2025, 11, 4, 9, 17, 37, 123507), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:17:38,007 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,012 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,043 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,052 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,077 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,079 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,100 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,245 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:38,251 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:17:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:05.435809+00:00 (in 26.998246 seconds) gcworker stdout | 2025-11-04 09:17:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:08 GMT)" (scheduled at 2025-11-04 09:17:38.437104+00:00) gcworker stdout | 2025-11-04 09:17:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:17:38,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247798449, None, 1, 0]) gcworker stdout | 2025-11-04 09:17:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:17:38,450 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:38,467 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:38,466 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:17:39,252 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:17:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:17:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:01.922827+00:00 (in 21.997774 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:17:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:39 GMT)" (scheduled at 2025-11-04 09:17:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:17:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:17:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:17:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:47.122849+00:00 (in 6.997981 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:40 GMT)" (scheduled at 2025-11-04 09:17:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:17:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:17:41,010 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:51.530064+00:00 (in 9.999475 seconds) notificationworker stdout | 2025-11-04 09:17:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:51 GMT)" (scheduled at 2025-11-04 09:17:41.530064+00:00) notificationworker stdout | 2025-11-04 09:17:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 41, 530858), True, datetime.datetime(2025, 11, 4, 9, 17, 41, 530858), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:17:42,456 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:17:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:17:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:57.523915+00:00 (in 14.090673 seconds) repositorygcworker stdout | 2025-11-04 09:17:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:43 GMT)" (scheduled at 2025-11-04 09:17:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:17:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:17:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 43, 433484), True, datetime.datetime(2025, 11, 4, 9, 17, 43, 433484), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:17:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:17:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:17:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:17:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:17:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:59.351216+00:00 (in 14.999550 seconds) quotatotalworker stdout | 2025-11-04 09:17:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:59 GMT)" (scheduled at 2025-11-04 09:17:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:17:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:17:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:17:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:17:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:17:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:17:44,958 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:45,445 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:17:45,459 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:17:45,459 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:17:45,469 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:17:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:47.479080+00:00 (in 1.998896 seconds) securityworker stdout | 2025-11-04 09:17:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:15 GMT)" (scheduled at 2025-11-04 09:17:45.479769+00:00) securityworker stdout | 2025-11-04 09:17:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 45, 483712), 1, 2]) securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:45,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 45, 483712), 1, 2]) securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:17:45,697 [287] [DEBUG] [app] Starting request: urn:request:52277fbb-4f7c-4bea-87a6-3c195fc91512 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:45,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:45,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:45,701 [299] [DEBUG] [app] Starting request: urn:request:b1255ff4-51f7-45d7-9c44-c6d7cb3a4260 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:45,702 [299] [DEBUG] [app] Ending request: urn:request:b1255ff4-51f7-45d7-9c44-c6d7cb3a4260 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b1255ff4-51f7-45d7-9c44-c6d7cb3a4260', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:17:45,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:45,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:45,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:45,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:45,706 [285] [DEBUG] [app] Starting request: urn:request:1c11c0d5-b9d3-4370-bbe4-01d921630129 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:45,706 [285] [DEBUG] [app] Ending request: urn:request:1c11c0d5-b9d3-4370-bbe4-01d921630129 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1c11c0d5-b9d3-4370-bbe4-01d921630129', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:45,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:45,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:45,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:45,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:45,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:45,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:45,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:45,728 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:45,731 [287] [DEBUG] [app] Ending request: urn:request:52277fbb-4f7c-4bea-87a6-3c195fc91512 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:52277fbb-4f7c-4bea-87a6-3c195fc91512', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:45,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:45,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:17:45,784 [288] [DEBUG] [app] Starting request: urn:request:9b1780ee-7fef-491f-8581-1a03385c03df (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:17:45,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:45,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:17:45,788 [296] [DEBUG] [app] Starting request: urn:request:eae9b578-8782-4375-bad6-d01d480d923c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:17:45,789 [296] [DEBUG] [app] Ending request: urn:request:eae9b578-8782-4375-bad6-d01d480d923c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:eae9b578-8782-4375-bad6-d01d480d923c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:17:45,789 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:45,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:45,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:45,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:17:45,793 [285] [DEBUG] [app] Starting request: urn:request:c12eeb4a-5c80-4dab-9e75-8e46351338b3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:17:45,793 [285] [DEBUG] [app] Ending request: urn:request:c12eeb4a-5c80-4dab-9e75-8e46351338b3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c12eeb4a-5c80-4dab-9e75-8e46351338b3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:17:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:17:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:45,794 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:45,794 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:45,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:45,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:45,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:45,815 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:45,818 [288] [DEBUG] [app] Ending request: urn:request:9b1780ee-7fef-491f-8581-1a03385c03df (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9b1780ee-7fef-491f-8581-1a03385c03df', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:45,818 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:45,818 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:17:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:17:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:17:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:57.122849+00:00 (in 9.999617 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:57 GMT)" (scheduled at 2025-11-04 09:17:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 47, 123531), True, datetime.datetime(2025, 11, 4, 9, 17, 47, 123531), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:17:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:15.479769+00:00 (in 28.000207 seconds) securityworker stdout | 2025-11-04 09:17:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:17 GMT)" (scheduled at 2025-11-04 09:17:47.479080+00:00) securityworker stdout | 2025-11-04 09:17:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:17:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:47,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 47, 483588), 1, 2]) securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:47,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:47,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:17:47,508 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:47,509 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 47, 483588), 1, 2]) securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:17:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:47,512 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:17:47,513 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:17:47,903 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:48,561 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:17:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:17:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:33.622985+00:00 (in 44.997405 seconds) exportactionlogsworker stdout | 2025-11-04 09:17:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:48 GMT)" (scheduled at 2025-11-04 09:17:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:17:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:17:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:17:49,880 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:17:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:17:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:21.226294+00:00 (in 29.999548 seconds) autopruneworker stdout | 2025-11-04 09:17:51,227 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:21 GMT)" (scheduled at 2025-11-04 09:17:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:17:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247811235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:17:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:17:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:17:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:01.530064+00:00 (in 9.999545 seconds) notificationworker stdout | 2025-11-04 09:17:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:01 GMT)" (scheduled at 2025-11-04 09:17:51.530064+00:00) notificationworker stdout | 2025-11-04 09:17:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 51, 530812), True, datetime.datetime(2025, 11, 4, 9, 17, 51, 530812), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:17:52,135 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:55,003 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:55,469 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:17:55,482 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:17:55,483 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:17:55,492 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:07.122849+00:00 (in 9.999631 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:07 GMT)" (scheduled at 2025-11-04 09:17:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 57, 123452), True, datetime.datetime(2025, 11, 4, 9, 17, 57, 123452), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:17:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:09.133645+00:00 (in 12.001428 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:57 GMT)" (scheduled at 2025-11-04 09:17:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:17:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:17:57,144 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:17:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:17:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:17:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:27.222846+00:00 (in 29.999640 seconds) buildlogsarchiver stdout | 2025-11-04 09:17:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:27 GMT)" (scheduled at 2025-11-04 09:17:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:17:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 17, 57, 223390), False, 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:17:57,232 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:17:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:17:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:17:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:17:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:17:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:17:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:17:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:43.432867+00:00 (in 45.908562 seconds) repositorygcworker stdout | 2025-11-04 09:17:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:57 GMT)" (scheduled at 2025-11-04 09:17:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:17:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:17:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:17:57,954 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:17:58,609 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:17:58,990 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:59,323 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:17:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:17:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:14.351216+00:00 (in 14.999535 seconds) quotatotalworker stdout | 2025-11-04 09:17:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:14 GMT)" (scheduled at 2025-11-04 09:17:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:17:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:17:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:17:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:17:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:17:59,542 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:59,691 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:59,709 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:59,796 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:00,697 [288] [DEBUG] [app] Starting request: urn:request:e395f762-ea8f-4b63-a649-07d0a1d4bf04 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:00,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:00,701 [292] [DEBUG] [app] Starting request: urn:request:105f3146-7a54-4c31-8a3b-ab77f9419751 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:00,702 [292] [DEBUG] [app] Ending request: urn:request:105f3146-7a54-4c31-8a3b-ab77f9419751 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:105f3146-7a54-4c31-8a3b-ab77f9419751', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:18:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:00,706 [287] [DEBUG] [app] Starting request: urn:request:bd374984-97f5-46e0-a1b3-7e0cac526c30 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:00,706 [287] [DEBUG] [app] Ending request: urn:request:bd374984-97f5-46e0-a1b3-7e0cac526c30 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bd374984-97f5-46e0-a1b3-7e0cac526c30', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:00,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:00,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:00,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:00,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:00,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:00,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:00,729 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:00,731 [288] [DEBUG] [app] Ending request: urn:request:e395f762-ea8f-4b63-a649-07d0a1d4bf04 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e395f762-ea8f-4b63-a649-07d0a1d4bf04', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:00,732 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:00,732 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:18:00,784 [288] [DEBUG] [app] Starting request: urn:request:ff5ff74c-dc30-4aad-b655-848f5d54eb79 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:00,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:00,785 [292] [DEBUG] [app] Starting request: urn:request:269967c8-72ae-4b13-acfb-97195a564949 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:00,785 [292] [DEBUG] [app] Ending request: urn:request:269967c8-72ae-4b13-acfb-97195a564949 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:269967c8-72ae-4b13-acfb-97195a564949', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:18:00,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:00,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:00,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:00,789 [287] [DEBUG] [app] Starting request: urn:request:f5ac1328-4499-4c38-9baa-82a768102a07 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:00,790 [287] [DEBUG] [app] Ending request: urn:request:f5ac1328-4499-4c38-9baa-82a768102a07 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f5ac1328-4499-4c38-9baa-82a768102a07', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:00,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:00,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:00,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:00,790 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:00,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:00,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:00,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:00,808 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:00,811 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:00,814 [288] [DEBUG] [app] Ending request: urn:request:ff5ff74c-dc30-4aad-b655-848f5d54eb79 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ff5ff74c-dc30-4aad-b655-848f5d54eb79', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:00,814 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) gunicorn-web stdout | 2025-11-04 09:18:00,815 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:18:01,237 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:11.530064+00:00 (in 9.999539 seconds) notificationworker stdout | 2025-11-04 09:18:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:11 GMT)" (scheduled at 2025-11-04 09:18:01.530064+00:00) notificationworker stdout | 2025-11-04 09:18:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 1, 530786), True, datetime.datetime(2025, 11, 4, 9, 18, 1, 530786), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:18:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:18:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:39.924652+00:00 (in 38.001414 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:18:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:01 GMT)" (scheduled at 2025-11-04 09:18:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:18:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:18:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 1, 923483), True, datetime.datetime(2025, 11, 4, 9, 18, 1, 923483), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:18:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:18:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:18:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:18:02,031 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:18:02,220 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:18:02,231 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:18:03,511 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:18:03,525 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:18:03,616 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:18:04,402 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:18:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:08.437104+00:00 (in 3.000875 seconds) gcworker stdout | 2025-11-04 09:18:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:35 GMT)" (scheduled at 2025-11-04 09:18:05.435809+00:00) gcworker stdout | 2025-11-04 09:18:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038285436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:18:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:18:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:35 GMT)" executed successfully builder stdout | 2025-11-04 09:18:05,493 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:18:05,506 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:18:05,506 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:18:05,516 [66] [DEBUG] [data.database] Disconnecting from database. pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:18:05,704 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:18:06,073 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:18:06,696 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:18:07,089 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:17.122849+00:00 (in 9.999653 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:17 GMT)" (scheduled at 2025-11-04 09:18:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 7, 123472), True, datetime.datetime(2025, 11, 4, 9, 18, 7, 123472), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:18:08,019 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,027 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,054 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,066 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,091 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,091 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,111 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,258 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:08,265 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:18:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:18:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:21.434460+00:00 (in 12.998636 seconds) namespacegcworker stdout | 2025-11-04 09:18:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:23:08 GMT)" (scheduled at 2025-11-04 09:18:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:18:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 18, 8, 436043), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:18:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:35.435809+00:00 (in 26.998335 seconds) gcworker stdout | 2025-11-04 09:18:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:38 GMT)" (scheduled at 2025-11-04 09:18:08.437104+00:00) gcworker stdout | 2025-11-04 09:18:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) namespacegcworker stdout | 2025-11-04 09:18:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 18, 8, 436043), True, datetime.datetime(2025, 11, 4, 9, 18, 8, 436043), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:18:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247828449, None, 1, 0]) namespacegcworker stdout | 2025-11-04 09:18:08,451 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 18, 8, 436043), True, datetime.datetime(2025, 11, 4, 9, 18, 8, 436043), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 18, 8, 436043), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:18:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:18:08,455 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:18:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:23:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:18:08,475 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:08,480 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:08,489 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:18:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:18:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:57.131795+00:00 (in 47.997682 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:09 GMT)" (scheduled at 2025-11-04 09:18:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:18:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:18:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:18:09,260 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:18:11,019 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:15.535095+00:00 (in 4.004634 seconds) notificationworker stdout | 2025-11-04 09:18:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:21 GMT)" (scheduled at 2025-11-04 09:18:11.530064+00:00) notificationworker stdout | 2025-11-04 09:18:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 11, 530717), True, datetime.datetime(2025, 11, 4, 9, 18, 11, 530717), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:18:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:18:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:12.222828+00:00 (in 59.999585 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:18:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:12 GMT)" (scheduled at 2025-11-04 09:18:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:18:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:18:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:18:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:18:12,472 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:18:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:18:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:29.351216+00:00 (in 14.999583 seconds) quotatotalworker stdout | 2025-11-04 09:18:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:29 GMT)" (scheduled at 2025-11-04 09:18:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:18:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:18:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:18:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:18:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:18:14,967 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:18:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:17.479080+00:00 (in 1.998864 seconds) securityworker stdout | 2025-11-04 09:18:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:45 GMT)" (scheduled at 2025-11-04 09:18:15.479769+00:00) securityworker stdout | 2025-11-04 09:18:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 15, 483798), 1, 2]) securityworker stdout | 2025-11-04 09:18:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 15, 483798), 1, 2]) securityworker stdout | 2025-11-04 09:18:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:45 GMT)" executed successfully builder stdout | 2025-11-04 09:18:15,516 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:18:15,528 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:18:15,528 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics notificationworker stdout | 2025-11-04 09:18:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:21.530064+00:00 (in 5.994528 seconds) notificationworker stdout | 2025-11-04 09:18:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:15 GMT)" (scheduled at 2025-11-04 09:18:15.535095+00:00) notificationworker stdout | 2025-11-04 09:18:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:18:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:15 GMT)" executed successfully builder stdout | 2025-11-04 09:18:15,539 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:15,697 [287] [DEBUG] [app] Starting request: urn:request:ede191f8-0471-4ed1-be17-1bbb321cf804 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:15,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:15,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:15,701 [298] [DEBUG] [app] Starting request: urn:request:9e91a37f-a558-478b-b2e1-50d3399d3f19 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:15,701 [298] [DEBUG] [app] Ending request: urn:request:9e91a37f-a558-478b-b2e1-50d3399d3f19 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9e91a37f-a558-478b-b2e1-50d3399d3f19', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:18:15,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:15,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:15,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:15,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:15,705 [285] [DEBUG] [app] Starting request: urn:request:19e7292c-8130-4d8c-abbf-516d3a6fbb31 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:15,706 [285] [DEBUG] [app] Ending request: urn:request:19e7292c-8130-4d8c-abbf-516d3a6fbb31 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:19e7292c-8130-4d8c-abbf-516d3a6fbb31', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:15,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:15,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:15,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:15,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:15,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:15,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:15,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:15,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:15,730 [287] [DEBUG] [app] Ending request: urn:request:ede191f8-0471-4ed1-be17-1bbb321cf804 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ede191f8-0471-4ed1-be17-1bbb321cf804', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:15,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:15,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:18:15,784 [285] [DEBUG] [app] Starting request: urn:request:64940d8e-d963-401e-8ea8-219284a67ac3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:15,788 [299] [DEBUG] [app] Starting request: urn:request:e5e399ce-1743-4563-9ee3-877182b8e869 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:15,788 [299] [DEBUG] [app] Ending request: urn:request:e5e399ce-1743-4563-9ee3-877182b8e869 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e5e399ce-1743-4563-9ee3-877182b8e869', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:18:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:15,792 [287] [DEBUG] [app] Starting request: urn:request:850797eb-86ff-48e4-aa58-e37091f06aad (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:15,792 [287] [DEBUG] [app] Ending request: urn:request:850797eb-86ff-48e4-aa58-e37091f06aad (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:850797eb-86ff-48e4-aa58-e37091f06aad', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:15,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:15,817 [285] [DEBUG] [app] Ending request: urn:request:64940d8e-d963-401e-8ea8-219284a67ac3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:64940d8e-d963-401e-8ea8-219284a67ac3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:18:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:27.122849+00:00 (in 9.999640 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:27 GMT)" (scheduled at 2025-11-04 09:18:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:17,123 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 17, 123446), True, datetime.datetime(2025, 11, 4, 9, 18, 17, 123446), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:18:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:45.479769+00:00 (in 28.000191 seconds) securityworker stdout | 2025-11-04 09:18:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:47 GMT)" (scheduled at 2025-11-04 09:18:17.479080+00:00) securityworker stdout | 2025-11-04 09:18:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:18:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:18:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:18:17,912 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:18,570 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:18:19,889 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:18:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:18:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:51.226294+00:00 (in 29.999561 seconds) autopruneworker stdout | 2025-11-04 09:18:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:51 GMT)" (scheduled at 2025-11-04 09:18:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:18:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247841234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:18:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:18:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:18:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:18:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:18:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:32.435813+00:00 (in 11.000971 seconds) namespacegcworker stdout | 2025-11-04 09:18:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:21 GMT)" (scheduled at 2025-11-04 09:18:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:18:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:18:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 21, 435134), True, datetime.datetime(2025, 11, 4, 9, 18, 21, 435134), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:18:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:18:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:18:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:31.530064+00:00 (in 9.999591 seconds) notificationworker stdout | 2025-11-04 09:18:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:31 GMT)" (scheduled at 2025-11-04 09:18:21.530064+00:00) notificationworker stdout | 2025-11-04 09:18:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 21, 530779), True, datetime.datetime(2025, 11, 4, 9, 18, 21, 530779), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:18:22,144 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:25,012 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:18:25,539 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:18:25,552 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:18:25,552 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:18:25,562 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:37.122849+00:00 (in 9.999634 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:37 GMT)" (scheduled at 2025-11-04 09:18:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 27, 123468), True, datetime.datetime(2025, 11, 4, 9, 18, 27, 123468), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:18:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:18:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:57.222846+00:00 (in 29.999626 seconds) buildlogsarchiver stdout | 2025-11-04 09:18:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:57 GMT)" (scheduled at 2025-11-04 09:18:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:18:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 18, 27, 223408), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:18:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:18:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:18:27,240 [67] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:18:27,241 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:18:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:18:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:18:27,963 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:18:28,618 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:18:28,999 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:29,333 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:18:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:18:29,352 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:44.351216+00:00 (in 14.999141 seconds) quotatotalworker stdout | 2025-11-04 09:18:29,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:44 GMT)" (scheduled at 2025-11-04 09:18:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:18:29,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:18:29,353 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:18:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:18:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:18:29,560 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:29,703 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:29,726 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:29,811 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:30,697 [288] [DEBUG] [app] Starting request: urn:request:4295fd4c-c24d-433b-a43d-82a10afcb9bc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:30,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:30,701 [298] [DEBUG] [app] Starting request: urn:request:ee0e79cd-89df-4898-9380-c1f1d6c22c37 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:30,701 [298] [DEBUG] [app] Ending request: urn:request:ee0e79cd-89df-4898-9380-c1f1d6c22c37 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ee0e79cd-89df-4898-9380-c1f1d6c22c37', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:30,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:18:30,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:30,706 [287] [DEBUG] [app] Starting request: urn:request:8d45ba31-0743-4f23-b02a-a6ccb672629c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:30,706 [287] [DEBUG] [app] Ending request: urn:request:8d45ba31-0743-4f23-b02a-a6ccb672629c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8d45ba31-0743-4f23-b02a-a6ccb672629c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:18:30,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:30,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:30,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:30,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:30,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:30,730 [288] [DEBUG] [app] Ending request: urn:request:4295fd4c-c24d-433b-a43d-82a10afcb9bc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4295fd4c-c24d-433b-a43d-82a10afcb9bc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:30,731 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:18:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:18:30,784 [287] [DEBUG] [app] Starting request: urn:request:7b83f795-1b8e-4e3e-bea4-a4b96e4d7590 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:30,784 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:30,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:30,788 [298] [DEBUG] [app] Starting request: urn:request:8daf7fa1-ce63-4000-84b3-9facb0fb3298 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:30,788 [298] [DEBUG] [app] Ending request: urn:request:8daf7fa1-ce63-4000-84b3-9facb0fb3298 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8daf7fa1-ce63-4000-84b3-9facb0fb3298', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:18:30,788 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:30,788 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:30,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:30,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:30,792 [288] [DEBUG] [app] Starting request: urn:request:ff2f78e6-aaf6-4f64-82ac-b9be1fc94773 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:30,792 [288] [DEBUG] [app] Ending request: urn:request:ff2f78e6-aaf6-4f64-82ac-b9be1fc94773 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ff2f78e6-aaf6-4f64-82ac-b9be1fc94773', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:30,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:30,792 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:30,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:30,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:30,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:30,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:30,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:30,810 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:30,813 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:30,816 [287] [DEBUG] [app] Ending request: urn:request:7b83f795-1b8e-4e3e-bea4-a4b96e4d7590 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7b83f795-1b8e-4e3e-bea4-a4b96e4d7590', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:30,816 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:30,816 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:31,246 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:41.530064+00:00 (in 9.999545 seconds) notificationworker stdout | 2025-11-04 09:18:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:41 GMT)" (scheduled at 2025-11-04 09:18:31.530064+00:00) notificationworker stdout | 2025-11-04 09:18:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 31, 530837), True, datetime.datetime(2025, 11, 4, 9, 18, 31, 530837), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:18:32,040 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:18:32,229 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:18:32,240 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:18:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:18:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:21.434460+00:00 (in 48.998255 seconds) namespacegcworker stdout | 2025-11-04 09:18:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:32 GMT)" (scheduled at 2025-11-04 09:18:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:18:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:18:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:18:33,531 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:18:33,534 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:18:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:18:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:48.625192+00:00 (in 15.001794 seconds) exportactionlogsworker stdout | 2025-11-04 09:18:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:33 GMT)" (scheduled at 2025-11-04 09:18:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:18:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:18:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 33, 623706), True, datetime.datetime(2025, 11, 4, 9, 18, 33, 623706), 0, 'exportactionlogs/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:18:33,626 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:18:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:18:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:18:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:33 GMT)" executed successfully builder stdout | 2025-11-04 09:18:34,411 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:18:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:38.437104+00:00 (in 3.000902 seconds) gcworker stdout | 2025-11-04 09:18:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:05 GMT)" (scheduled at 2025-11-04 09:18:35.435809+00:00) gcworker stdout | 2025-11-04 09:18:35,436 [72] [DEBUG] [peewee] ('SELECT DISTINCT "t1"."removed_tag_expiration_s" FROM "user" AS "t1" LIMIT %s', [100]) gcworker stdout | 2025-11-04 09:18:35,448 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038315448, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:18:35,453 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:18:35,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:35,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:05 GMT)" executed successfully builder stdout | 2025-11-04 09:18:35,562 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:18:35,575 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:18:35,575 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:18:35,586 [66] [DEBUG] [data.database] Disconnecting from database. pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:18:35,713 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:18:36,082 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:18:36,705 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:18:37,098 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:40.124445+00:00 (in 3.001244 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:47 GMT)" (scheduled at 2025-11-04 09:18:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:37,123 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 37, 123458), True, datetime.datetime(2025, 11, 4, 9, 18, 37, 123458), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:37,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:18:38,030 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,041 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,064 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,080 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,102 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,103 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,125 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,270 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:38,277 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:18:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:05.435809+00:00 (in 26.998327 seconds) gcworker stdout | 2025-11-04 09:18:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:08 GMT)" (scheduled at 2025-11-04 09:18:38.437104+00:00) gcworker stdout | 2025-11-04 09:18:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:18:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247858449, None, 1, 0]) gcworker stdout | 2025-11-04 09:18:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:18:38,491 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:38,495 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:38,499 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:18:39,269 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:18:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:18:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:01.922827+00:00 (in 21.997810 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:18:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:39 GMT)" (scheduled at 2025-11-04 09:18:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:18:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:18:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:18:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:47.122849+00:00 (in 6.998039 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:40 GMT)" (scheduled at 2025-11-04 09:18:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:18:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:18:41,028 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:51.530064+00:00 (in 9.999482 seconds) notificationworker stdout | 2025-11-04 09:18:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:51 GMT)" (scheduled at 2025-11-04 09:18:41.530064+00:00) notificationworker stdout | 2025-11-04 09:18:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 41, 530929), True, datetime.datetime(2025, 11, 4, 9, 18, 41, 530929), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:18:42,480 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:18:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:18:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:57.523915+00:00 (in 14.090668 seconds) repositorygcworker stdout | 2025-11-04 09:18:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:43 GMT)" (scheduled at 2025-11-04 09:18:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:18:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:18:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 43, 433513), True, datetime.datetime(2025, 11, 4, 9, 18, 43, 433513), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:18:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:18:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:18:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:18:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:18:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:59.351216+00:00 (in 14.999544 seconds) quotatotalworker stdout | 2025-11-04 09:18:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:59 GMT)" (scheduled at 2025-11-04 09:18:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:18:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:18:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:18:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:18:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:18:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:18:44,986 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:18:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:47.479080+00:00 (in 1.998897 seconds) securityworker stdout | 2025-11-04 09:18:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:15 GMT)" (scheduled at 2025-11-04 09:18:45.479769+00:00) securityworker stdout | 2025-11-04 09:18:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 45, 483591), 1, 2]) securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 45, 483591), 1, 2]) securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:15 GMT)" executed successfully builder stdout | 2025-11-04 09:18:45,586 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:18:45,598 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:18:45,598 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:18:45,608 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:45,697 [288] [DEBUG] [app] Starting request: urn:request:4a149898-469e-4fa8-9b4f-b4c23a0bede7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:45,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:45,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:45,701 [292] [DEBUG] [app] Starting request: urn:request:9abef6db-c3c9-4ba8-aad6-1b419f2b28e8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:45,702 [292] [DEBUG] [app] Ending request: urn:request:9abef6db-c3c9-4ba8-aad6-1b419f2b28e8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9abef6db-c3c9-4ba8-aad6-1b419f2b28e8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:18:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:45,706 [286] [DEBUG] [app] Starting request: urn:request:c62d745a-3c7d-4f15-8e62-97a026a5d8af (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:45,706 [286] [DEBUG] [app] Ending request: urn:request:c62d745a-3c7d-4f15-8e62-97a026a5d8af (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c62d745a-3c7d-4f15-8e62-97a026a5d8af', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:45,706 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:45,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:45,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:45,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:45,731 [288] [DEBUG] [app] Ending request: urn:request:4a149898-469e-4fa8-9b4f-b4c23a0bede7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4a149898-469e-4fa8-9b4f-b4c23a0bede7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:45,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:45,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:18:45,783 [288] [DEBUG] [app] Starting request: urn:request:2da18b91-c54d-462c-b68f-a53fb6f8651f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:18:45,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:18:45,784 [292] [DEBUG] [app] Starting request: urn:request:2a07970d-348f-447b-ba43-e9b3881e9380 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:18:45,785 [292] [DEBUG] [app] Ending request: urn:request:2a07970d-348f-447b-ba43-e9b3881e9380 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2a07970d-348f-447b-ba43-e9b3881e9380', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:45,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:18:45,785 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:45,786 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:45,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:18:45,789 [287] [DEBUG] [app] Starting request: urn:request:f432bf76-1cf6-4c31-a063-d949a9b4808e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:18:45,789 [287] [DEBUG] [app] Ending request: urn:request:f432bf76-1cf6-4c31-a063-d949a9b4808e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f432bf76-1cf6-4c31-a063-d949a9b4808e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:45,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:18:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:18:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:45,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:45,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:45,790 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:45,790 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:45,797 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:45,797 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:45,807 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:45,810 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:45,813 [288] [DEBUG] [app] Ending request: urn:request:2da18b91-c54d-462c-b68f-a53fb6f8651f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2da18b91-c54d-462c-b68f-a53fb6f8651f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:45,813 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:45,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:18:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:18:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:18:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:57.122849+00:00 (in 9.999622 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:57 GMT)" (scheduled at 2025-11-04 09:18:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 47, 123467), True, datetime.datetime(2025, 11, 4, 9, 18, 47, 123467), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:18:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:15.479769+00:00 (in 28.000286 seconds) securityworker stdout | 2025-11-04 09:18:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:17 GMT)" (scheduled at 2025-11-04 09:18:47.479080+00:00) securityworker stdout | 2025-11-04 09:18:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:18:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:47,501 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 47, 483059), 1, 2]) securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:47,505 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 47, 483059), 1, 2]) securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:47,509 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:18:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:47,509 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:18:47,921 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:48,579 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:18:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:18:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:33.622985+00:00 (in 44.997403 seconds) exportactionlogsworker stdout | 2025-11-04 09:18:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:48 GMT)" (scheduled at 2025-11-04 09:18:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:18:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:18:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:18:49,898 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:18:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:18:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:21.226294+00:00 (in 29.999549 seconds) autopruneworker stdout | 2025-11-04 09:18:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:21 GMT)" (scheduled at 2025-11-04 09:18:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:18:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247871234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:18:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:18:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:18:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:01.530064+00:00 (in 9.999526 seconds) notificationworker stdout | 2025-11-04 09:18:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:01 GMT)" (scheduled at 2025-11-04 09:18:51.530064+00:00) notificationworker stdout | 2025-11-04 09:18:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 51, 530813), True, datetime.datetime(2025, 11, 4, 9, 18, 51, 530813), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:18:52,153 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:55,021 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:18:55,609 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:18:55,622 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:18:55,622 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:18:55,632 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:07.122849+00:00 (in 9.999635 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:07 GMT)" (scheduled at 2025-11-04 09:18:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 57, 123469), True, datetime.datetime(2025, 11, 4, 9, 18, 57, 123469), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:18:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:09.133645+00:00 (in 12.001433 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:57 GMT)" (scheduled at 2025-11-04 09:18:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:18:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:18:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:18:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:18:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:18:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:27.222846+00:00 (in 29.999643 seconds) buildlogsarchiver stdout | 2025-11-04 09:18:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:27 GMT)" (scheduled at 2025-11-04 09:18:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:18:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 18, 57, 223727), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:18:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:18:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:18:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:18:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:18:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:18:57,250 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:18:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:18:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:02.523412+00:00 (in 4.999074 seconds) repositorygcworker stdout | 2025-11-04 09:18:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:57 GMT)" (scheduled at 2025-11-04 09:18:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:18:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:18:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:18:57,972 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:18:58,627 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:18:59,008 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:59,345 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:18:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:18:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:14.351216+00:00 (in 14.999548 seconds) quotatotalworker stdout | 2025-11-04 09:18:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:14 GMT)" (scheduled at 2025-11-04 09:18:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:18:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:18:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:18:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:18:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:18:59,572 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:59,717 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:59,741 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:59,825 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:00,697 [287] [DEBUG] [app] Starting request: urn:request:81062f68-91cf-43e3-826c-9e9a1b790df6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:00,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:00,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:00,701 [298] [DEBUG] [app] Starting request: urn:request:17d49316-01e0-4352-86d2-b8b24bfa889f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:00,702 [298] [DEBUG] [app] Ending request: urn:request:17d49316-01e0-4352-86d2-b8b24bfa889f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:17d49316-01e0-4352-86d2-b8b24bfa889f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:19:00,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:00,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:00,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:00,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:00,706 [285] [DEBUG] [app] Starting request: urn:request:529f85cb-54a4-457d-b5c7-d93caf830774 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:00,706 [285] [DEBUG] [app] Ending request: urn:request:529f85cb-54a4-457d-b5c7-d93caf830774 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:529f85cb-54a4-457d-b5c7-d93caf830774', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:00,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:00,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:00,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:00,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:00,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:00,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:00,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:00,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:00,730 [287] [DEBUG] [app] Ending request: urn:request:81062f68-91cf-43e3-826c-9e9a1b790df6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:81062f68-91cf-43e3-826c-9e9a1b790df6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:00,730 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:19:00,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:19:00,784 [285] [DEBUG] [app] Starting request: urn:request:2414a375-c3e4-4055-8e04-37418fd1f931 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:00,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:00,788 [299] [DEBUG] [app] Starting request: urn:request:bdee0b7f-ac09-4534-8ae3-b6e86973ba25 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:00,788 [299] [DEBUG] [app] Ending request: urn:request:bdee0b7f-ac09-4534-8ae3-b6e86973ba25 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bdee0b7f-ac09-4534-8ae3-b6e86973ba25', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:19:00,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:00,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:00,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:00,793 [287] [DEBUG] [app] Starting request: urn:request:6794c31a-4647-4a30-8405-ba68e5ad59a3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:00,793 [287] [DEBUG] [app] Ending request: urn:request:6794c31a-4647-4a30-8405-ba68e5ad59a3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6794c31a-4647-4a30-8405-ba68e5ad59a3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:00,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:19:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:00,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:00,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:00,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:00,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:00,817 [285] [DEBUG] [app] Ending request: urn:request:2414a375-c3e4-4055-8e04-37418fd1f931 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2414a375-c3e4-4055-8e04-37418fd1f931', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:00,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:19:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:19:01,255 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:11.530064+00:00 (in 9.999604 seconds) notificationworker stdout | 2025-11-04 09:19:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:11 GMT)" (scheduled at 2025-11-04 09:19:01.530064+00:00) notificationworker stdout | 2025-11-04 09:19:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 1, 530730), True, datetime.datetime(2025, 11, 4, 9, 19, 1, 530730), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:19:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:19:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:39.924652+00:00 (in 38.001436 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:19:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:01 GMT)" (scheduled at 2025-11-04 09:19:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:19:01,924 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:19:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 1, 924078), True, datetime.datetime(2025, 11, 4, 9, 19, 1, 924078), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:19:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:19:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:19:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:19:02,049 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:19:02,238 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:19:02,249 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:19:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:19:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:43.432867+00:00 (in 40.908957 seconds) repositorygcworker stdout | 2025-11-04 09:19:02,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:02 GMT)" (scheduled at 2025-11-04 09:19:02.523412+00:00) repositorygcworker stdout | 2025-11-04 09:19:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 19, 2, 524212), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:19:02,536 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 2, 524212), True, datetime.datetime(2025, 11, 4, 9, 19, 2, 524212), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:19:02,550 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 2, 524212), True, datetime.datetime(2025, 11, 4, 9, 19, 2, 524212), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 19, 2, 524212), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:19:02,554 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:19:02,554 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:19:03,540 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:19:03,543 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:19:03,634 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:19:04,421 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:19:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:08.437104+00:00 (in 3.000906 seconds) gcworker stdout | 2025-11-04 09:19:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:35 GMT)" (scheduled at 2025-11-04 09:19:05.435809+00:00) gcworker stdout | 2025-11-04 09:19:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038345436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:19:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:19:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:35 GMT)" executed successfully builder stdout | 2025-11-04 09:19:05,632 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:19:05,646 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:19:05,646 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:19:05,656 [66] [DEBUG] [data.database] Disconnecting from database. pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:19:05,722 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:19:06,091 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:19:06,714 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:19:07,107 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:17.122849+00:00 (in 9.999647 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:17 GMT)" (scheduled at 2025-11-04 09:19:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 7, 123455), True, datetime.datetime(2025, 11, 4, 9, 19, 7, 123455), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:17 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:19:08,040 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,054 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,074 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,091 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,114 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,116 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,154 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,281 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:08,296 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:19:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:35.435809+00:00 (in 26.998332 seconds) gcworker stdout | 2025-11-04 09:19:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:38 GMT)" (scheduled at 2025-11-04 09:19:08.437104+00:00) gcworker stdout | 2025-11-04 09:19:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:19:08,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247888449, None, 1, 0]) gcworker stdout | 2025-11-04 09:19:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:38 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:19:08,505 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:08,509 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:08,510 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:19:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:19:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:57.131795+00:00 (in 47.997740 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:09 GMT)" (scheduled at 2025-11-04 09:19:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:19:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:19:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:19:09,278 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:19:11,036 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:15.535095+00:00 (in 4.004580 seconds) notificationworker stdout | 2025-11-04 09:19:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:21 GMT)" (scheduled at 2025-11-04 09:19:11.530064+00:00) notificationworker stdout | 2025-11-04 09:19:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 11, 530833), True, datetime.datetime(2025, 11, 4, 9, 19, 11, 530833), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:19:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:19:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:12.222828+00:00 (in 59.999588 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:19:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:12 GMT)" (scheduled at 2025-11-04 09:19:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:19:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:19:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:19:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:19:12,489 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:19:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:19:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:29.351216+00:00 (in 14.999550 seconds) quotatotalworker stdout | 2025-11-04 09:19:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:29 GMT)" (scheduled at 2025-11-04 09:19:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:19:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:19:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:19:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:19:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:19:14,995 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:19:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:17.479080+00:00 (in 1.998868 seconds) securityworker stdout | 2025-11-04 09:19:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:45 GMT)" (scheduled at 2025-11-04 09:19:15.479769+00:00) securityworker stdout | 2025-11-04 09:19:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:19:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:19:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 15, 484016), 1, 2]) securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 15, 484016), 1, 2]) securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:21.530064+00:00 (in 5.994474 seconds) notificationworker stdout | 2025-11-04 09:19:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:15 GMT)" (scheduled at 2025-11-04 09:19:15.535095+00:00) notificationworker stdout | 2025-11-04 09:19:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:19:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:15 GMT)" executed successfully builder stdout | 2025-11-04 09:19:15,657 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:19:15,670 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:19:15,670 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:19:15,680 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:15,697 [288] [DEBUG] [app] Starting request: urn:request:67fbd3b6-c754-4a76-b4c8-a3d48444ee84 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:15,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:15,701 [299] [DEBUG] [app] Starting request: urn:request:5d712455-2bb4-409d-9b52-8df85770c62b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:15,702 [299] [DEBUG] [app] Ending request: urn:request:5d712455-2bb4-409d-9b52-8df85770c62b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5d712455-2bb4-409d-9b52-8df85770c62b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:19:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:15,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:15,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:15,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:15,706 [286] [DEBUG] [app] Starting request: urn:request:e1574842-c215-464c-972c-cb695f7513fc (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:15,706 [286] [DEBUG] [app] Ending request: urn:request:e1574842-c215-464c-972c-cb695f7513fc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e1574842-c215-464c-972c-cb695f7513fc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:15,706 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:15,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:15,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:15,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:15,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:15,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:15,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:15,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:15,730 [288] [DEBUG] [app] Ending request: urn:request:67fbd3b6-c754-4a76-b4c8-a3d48444ee84 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:67fbd3b6-c754-4a76-b4c8-a3d48444ee84', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:15,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:15,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:19:15,783 [285] [DEBUG] [app] Starting request: urn:request:232419e9-ba1c-413a-9ee6-be1c22a3d1de (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:15,788 [299] [DEBUG] [app] Starting request: urn:request:2fcca350-62cc-4c7f-ac93-3e569e5fffc3 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:15,788 [299] [DEBUG] [app] Ending request: urn:request:2fcca350-62cc-4c7f-ac93-3e569e5fffc3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2fcca350-62cc-4c7f-ac93-3e569e5fffc3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:19:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:15,792 [288] [DEBUG] [app] Starting request: urn:request:be6b162b-4473-4d3d-af33-926131d88fde (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:15,792 [288] [DEBUG] [app] Ending request: urn:request:be6b162b-4473-4d3d-af33-926131d88fde (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:be6b162b-4473-4d3d-af33-926131d88fde', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:15,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:15,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:15,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:15,817 [285] [DEBUG] [app] Ending request: urn:request:232419e9-ba1c-413a-9ee6-be1c22a3d1de (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:232419e9-ba1c-413a-9ee6-be1c22a3d1de', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:19:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:27.122849+00:00 (in 9.999639 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:27 GMT)" (scheduled at 2025-11-04 09:19:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 17, 123472), True, datetime.datetime(2025, 11, 4, 9, 19, 17, 123472), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:17,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:17,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:17,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:19:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:45.479769+00:00 (in 28.000202 seconds) securityworker stdout | 2025-11-04 09:19:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:47 GMT)" (scheduled at 2025-11-04 09:19:17.479080+00:00) securityworker stdout | 2025-11-04 09:19:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:19:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:19:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:19:17,930 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:18,588 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:19:19,907 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:19:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:19:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:51.226294+00:00 (in 29.999575 seconds) autopruneworker stdout | 2025-11-04 09:19:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:51 GMT)" (scheduled at 2025-11-04 09:19:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:19:21,234 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247901234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:19:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:19:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:19:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:19:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:19:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:32.435813+00:00 (in 11.000960 seconds) namespacegcworker stdout | 2025-11-04 09:19:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:21 GMT)" (scheduled at 2025-11-04 09:19:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:19:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:19:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 21, 435145), True, datetime.datetime(2025, 11, 4, 9, 19, 21, 435145), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:19:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:19:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:19:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:31.530064+00:00 (in 9.999556 seconds) notificationworker stdout | 2025-11-04 09:19:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:31 GMT)" (scheduled at 2025-11-04 09:19:21.530064+00:00) notificationworker stdout | 2025-11-04 09:19:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 21, 530747), True, datetime.datetime(2025, 11, 4, 9, 19, 21, 530747), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:19:22,166 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:25,030 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:19:25,680 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:19:25,692 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:19:25,692 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:19:25,702 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:37.122849+00:00 (in 9.999630 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:37 GMT)" (scheduled at 2025-11-04 09:19:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 27, 123480), True, datetime.datetime(2025, 11, 4, 9, 19, 27, 123480), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:19:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:19:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:57.222846+00:00 (in 29.999587 seconds) buildlogsarchiver stdout | 2025-11-04 09:19:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:57 GMT)" (scheduled at 2025-11-04 09:19:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:19:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 19, 27, 223523), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:19:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:19:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:19:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:19:27,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:19:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:19:27,259 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:19:27,981 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:19:28,636 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:19:29,017 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:19:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:19:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:44.351216+00:00 (in 14.999550 seconds) quotatotalworker stdout | 2025-11-04 09:19:29,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:44 GMT)" (scheduled at 2025-11-04 09:19:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:19:29,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:19:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) gunicorn-web stdout | 2025-11-04 09:19:29,356 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:19:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:19:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:19:29,584 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:29,739 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:29,754 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:29,847 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:30,696 [288] [DEBUG] [app] Starting request: urn:request:8f402055-be6d-49bb-9187-b01ba38717c9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:30,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:30,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:30,701 [292] [DEBUG] [app] Starting request: urn:request:1323fea4-d51d-4a1e-a3ff-0359956a7fff (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:30,701 [292] [DEBUG] [app] Ending request: urn:request:1323fea4-d51d-4a1e-a3ff-0359956a7fff (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1323fea4-d51d-4a1e-a3ff-0359956a7fff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:30,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:19:30,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:30,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:30,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:30,705 [285] [DEBUG] [app] Starting request: urn:request:9fcab090-2d79-4995-9a70-dc59071ddf47 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:30,705 [285] [DEBUG] [app] Ending request: urn:request:9fcab090-2d79-4995-9a70-dc59071ddf47 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9fcab090-2d79-4995-9a70-dc59071ddf47', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:30,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:30,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:30,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:30,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:30,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:30,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:30,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:30,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:30,730 [288] [DEBUG] [app] Ending request: urn:request:8f402055-be6d-49bb-9187-b01ba38717c9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8f402055-be6d-49bb-9187-b01ba38717c9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:30,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:30,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:19:30,783 [288] [DEBUG] [app] Starting request: urn:request:c7097369-c4a3-4ea8-b428-bea1b2bd5c3b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:30,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:30,785 [298] [DEBUG] [app] Starting request: urn:request:01a5e4c9-2387-4c2e-84b6-923af7948061 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:30,785 [298] [DEBUG] [app] Ending request: urn:request:01a5e4c9-2387-4c2e-84b6-923af7948061 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:01a5e4c9-2387-4c2e-84b6-923af7948061', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:30,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:19:30,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:30,786 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:30,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:30,789 [285] [DEBUG] [app] Starting request: urn:request:8c2c9431-bd1d-485a-8222-4554335c7fa7 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:30,789 [285] [DEBUG] [app] Ending request: urn:request:8c2c9431-bd1d-485a-8222-4554335c7fa7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8c2c9431-bd1d-485a-8222-4554335c7fa7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:30,790 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:30,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:30,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:30,790 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:30,790 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:30,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:30,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:30,807 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:30,811 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:30,814 [288] [DEBUG] [app] Ending request: urn:request:c7097369-c4a3-4ea8-b428-bea1b2bd5c3b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c7097369-c4a3-4ea8-b428-bea1b2bd5c3b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:30,814 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:30,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:31,263 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:40.533243+00:00 (in 9.002777 seconds) notificationworker stdout | 2025-11-04 09:19:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:41 GMT)" (scheduled at 2025-11-04 09:19:31.530064+00:00) notificationworker stdout | 2025-11-04 09:19:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 31, 530756), True, datetime.datetime(2025, 11, 4, 9, 19, 31, 530756), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:19:32,058 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:19:32,258 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:19:32,258 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:19:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:19:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:21.434460+00:00 (in 48.998266 seconds) namespacegcworker stdout | 2025-11-04 09:19:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:32 GMT)" (scheduled at 2025-11-04 09:19:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:19:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:19:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:19:33,548 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:19:33,551 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:19:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:19:33,624 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:48.625192+00:00 (in 15.001145 seconds) exportactionlogsworker stdout | 2025-11-04 09:19:33,624 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:33 GMT)" (scheduled at 2025-11-04 09:19:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:19:33,624 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:19:33,625 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 33, 624335), True, datetime.datetime(2025, 11, 4, 9, 19, 33, 624335), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:19:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:19:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:19:33,637 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:33 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:19:33,644 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:19:34,430 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:19:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:38.437104+00:00 (in 3.000885 seconds) gcworker stdout | 2025-11-04 09:19:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:05 GMT)" (scheduled at 2025-11-04 09:19:35.435809+00:00) gcworker stdout | 2025-11-04 09:19:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038375436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:19:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:19:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: builder stdout | 2025-11-04 09:19:35,703 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:19:35,716 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:19:35,717 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics queuecleanupworker stdout | 2025-11-04 09:19:35,731 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:19:35,736 [66] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:19:36,101 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:19:36,724 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:40.124445+00:00 (in 3.001196 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:47 GMT)" (scheduled at 2025-11-04 09:19:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 37, 123546), True, datetime.datetime(2025, 11, 4, 9, 19, 37, 123546), 0, 'proxycacheblob/%', 50, 1, 0]) quotatotalworker stdout | 2025-11-04 09:19:37,116 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:47 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:19:38,065 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,066 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,087 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,102 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,124 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,128 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,169 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,293 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:38,306 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:19:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:05.435809+00:00 (in 26.998334 seconds) gcworker stdout | 2025-11-04 09:19:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:08 GMT)" (scheduled at 2025-11-04 09:19:38.437104+00:00) gcworker stdout | 2025-11-04 09:19:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:19:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247918449, None, 1, 0]) gcworker stdout | 2025-11-04 09:19:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:19:38,517 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:38,521 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:38,520 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:19:39,287 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:19:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:19:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:01.922827+00:00 (in 21.997755 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:19:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:39 GMT)" (scheduled at 2025-11-04 09:19:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:19:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:19:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:19:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:47.122849+00:00 (in 6.998030 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:40 GMT)" (scheduled at 2025-11-04 09:19:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:19:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:41.530064+00:00 (in 0.996359 seconds) notificationworker stdout | 2025-11-04 09:19:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:40 GMT)" (scheduled at 2025-11-04 09:19:40.533243+00:00) notificationworker stdout | 2025-11-04 09:19:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 19, 40, 533948), 'notification/%']) notificationworker stdout | 2025-11-04 09:19:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 40, 533948), True, datetime.datetime(2025, 11, 4, 9, 19, 40, 533948), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:19:40,550 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 40, 533948), True, datetime.datetime(2025, 11, 4, 9, 19, 40, 533948), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 19, 40, 533948), 'notification/%']) notificationworker stdout | 2025-11-04 09:19:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:19:41,045 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:51.530064+00:00 (in 9.999630 seconds) notificationworker stdout | 2025-11-04 09:19:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:51 GMT)" (scheduled at 2025-11-04 09:19:41.530064+00:00) notificationworker stdout | 2025-11-04 09:19:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 41, 530703), True, datetime.datetime(2025, 11, 4, 9, 19, 41, 530703), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:19:42,498 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:19:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:19:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:57.523915+00:00 (in 14.090627 seconds) repositorygcworker stdout | 2025-11-04 09:19:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:43 GMT)" (scheduled at 2025-11-04 09:19:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:19:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:19:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 43, 433545), True, datetime.datetime(2025, 11, 4, 9, 19, 43, 433545), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:19:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:19:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:19:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:19:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:19:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:59.351216+00:00 (in 14.999606 seconds) quotatotalworker stdout | 2025-11-04 09:19:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:59 GMT)" (scheduled at 2025-11-04 09:19:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:19:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:19:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:19:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:19:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:19:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:19:45,004 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:19:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:47.479080+00:00 (in 1.998892 seconds) securityworker stdout | 2025-11-04 09:19:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:15 GMT)" (scheduled at 2025-11-04 09:19:45.479769+00:00) securityworker stdout | 2025-11-04 09:19:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:19:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 45, 483533), 1, 2]) securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 45, 483533), 1, 2]) securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:19:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:19:45,697 [286] [DEBUG] [app] Starting request: urn:request:ddcd449b-7ac6-44b0-8e3a-d72e78a38cbc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:45,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:45,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:45,701 [292] [DEBUG] [app] Starting request: urn:request:0320cb3e-9590-4183-be50-b2f15b97b835 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:45,702 [292] [DEBUG] [app] Ending request: urn:request:0320cb3e-9590-4183-be50-b2f15b97b835 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0320cb3e-9590-4183-be50-b2f15b97b835', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:19:45,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:45,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:45,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:45,706 [287] [DEBUG] [app] Starting request: urn:request:e032f6b1-17d9-437d-8a5e-bbb7aea6be74 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:45,706 [287] [DEBUG] [app] Ending request: urn:request:e032f6b1-17d9-437d-8a5e-bbb7aea6be74 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e032f6b1-17d9-437d-8a5e-bbb7aea6be74', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:45,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:45,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:45,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:45,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:45,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:45,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:45,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:45,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:45,728 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:45,731 [286] [DEBUG] [app] Ending request: urn:request:ddcd449b-7ac6-44b0-8e3a-d72e78a38cbc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ddcd449b-7ac6-44b0-8e3a-d72e78a38cbc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:45,731 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:45,731 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) builder stdout | 2025-11-04 09:19:45,736 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:19:45,748 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:19:45,748 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:19:45,758 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:45,783 [286] [DEBUG] [app] Starting request: urn:request:4e8fb04f-2fbe-4bdb-91e6-554194eb4d60 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:19:45,784 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:19:45,785 [292] [DEBUG] [app] Starting request: urn:request:91c32318-4808-42a0-90de-b5575e278597 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:19:45,785 [292] [DEBUG] [app] Ending request: urn:request:91c32318-4808-42a0-90de-b5575e278597 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:91c32318-4808-42a0-90de-b5575e278597', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:45,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:19:45,785 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:45,787 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:45,789 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:19:45,790 [285] [DEBUG] [app] Starting request: urn:request:44766f63-735a-466a-b4a6-ea5efefe6f2e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:19:45,790 [285] [DEBUG] [app] Ending request: urn:request:44766f63-735a-466a-b4a6-ea5efefe6f2e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:44766f63-735a-466a-b4a6-ea5efefe6f2e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:45,790 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:19:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:19:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:45,790 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:45,791 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:45,791 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:45,791 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:45,798 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:45,798 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:45,808 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:45,811 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:45,814 [286] [DEBUG] [app] Ending request: urn:request:4e8fb04f-2fbe-4bdb-91e6-554194eb4d60 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4e8fb04f-2fbe-4bdb-91e6-554194eb4d60', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:45,814 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:19:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.032) gunicorn-web stdout | 2025-11-04 09:19:45,815 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:19:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:19:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:53.124163+00:00 (in 6.000906 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:57 GMT)" (scheduled at 2025-11-04 09:19:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 47, 123589), True, datetime.datetime(2025, 11, 4, 9, 19, 47, 123589), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:19:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:15.479769+00:00 (in 28.000286 seconds) securityworker stdout | 2025-11-04 09:19:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:17 GMT)" (scheduled at 2025-11-04 09:19:47.479080+00:00) securityworker stdout | 2025-11-04 09:19:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:19:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:19:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:19:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 47, 483055), 1, 2]) securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:19:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 47, 483055), 1, 2]) securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:19:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:19:47,939 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:48,597 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:19:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:19:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:33.622985+00:00 (in 44.997363 seconds) exportactionlogsworker stdout | 2025-11-04 09:19:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:48 GMT)" (scheduled at 2025-11-04 09:19:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:19:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:19:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:19:49,917 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:19:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:19:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:21.226294+00:00 (in 29.999559 seconds) autopruneworker stdout | 2025-11-04 09:19:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:21 GMT)" (scheduled at 2025-11-04 09:19:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:19:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247931235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:19:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:19:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:19:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:01.530064+00:00 (in 9.999595 seconds) notificationworker stdout | 2025-11-04 09:19:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:01 GMT)" (scheduled at 2025-11-04 09:19:51.530064+00:00) notificationworker stdout | 2025-11-04 09:19:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 51, 530756), True, datetime.datetime(2025, 11, 4, 9, 19, 51, 530756), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:19:52,198 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:57.122849+00:00 (in 3.998245 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:53 GMT)" (scheduled at 2025-11-04 09:19:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 19, 53, 124850), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:19:53,137 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 53, 124850), True, datetime.datetime(2025, 11, 4, 9, 19, 53, 124850), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:19:53,141 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 53, 124850), True, datetime.datetime(2025, 11, 4, 9, 19, 53, 124850), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 19, 53, 124850), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:19:53,144 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:53,144 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:55,048 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:19:55,759 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:19:55,772 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:19:55,772 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:19:55,782 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:07.122849+00:00 (in 9.999605 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:07 GMT)" (scheduled at 2025-11-04 09:19:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 57, 123531), True, datetime.datetime(2025, 11, 4, 9, 19, 57, 123531), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:19:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:09.133645+00:00 (in 12.001433 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:57 GMT)" (scheduled at 2025-11-04 09:19:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:19:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:19:57,144 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:19:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:19:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:19:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:27.222846+00:00 (in 29.999331 seconds) buildlogsarchiver stdout | 2025-11-04 09:19:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:27 GMT)" (scheduled at 2025-11-04 09:19:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:19:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 19, 57, 223769), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:19:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:19:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:19:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:19:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:19:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:19:57,268 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:19:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:19:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:43.432867+00:00 (in 45.908537 seconds) repositorygcworker stdout | 2025-11-04 09:19:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:57 GMT)" (scheduled at 2025-11-04 09:19:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:19:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:19:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:19:57,989 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:19:58,645 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:19:59,025 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:19:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:19:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:14.351216+00:00 (in 14.999511 seconds) quotatotalworker stdout | 2025-11-04 09:19:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:14 GMT)" (scheduled at 2025-11-04 09:19:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:19:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:19:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:19:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:19:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:19:59,367 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:59,596 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:59,752 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:59,768 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:59,868 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:00,697 [288] [DEBUG] [app] Starting request: urn:request:d419fa21-4a30-4b23-bd43-1f9076093f94 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:00,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:00,702 [292] [DEBUG] [app] Starting request: urn:request:bb2cda59-93bd-4562-8aa1-7990e05d2de5 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:00,702 [292] [DEBUG] [app] Ending request: urn:request:bb2cda59-93bd-4562-8aa1-7990e05d2de5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bb2cda59-93bd-4562-8aa1-7990e05d2de5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:20:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:00,703 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:00,706 [286] [DEBUG] [app] Starting request: urn:request:143543b3-6dc1-49b1-946a-edec02f0f8ff (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:00,706 [286] [DEBUG] [app] Ending request: urn:request:143543b3-6dc1-49b1-946a-edec02f0f8ff (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:143543b3-6dc1-49b1-946a-edec02f0f8ff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:00,707 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:00,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:00,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:00,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:00,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:00,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:00,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:00,731 [288] [DEBUG] [app] Ending request: urn:request:d419fa21-4a30-4b23-bd43-1f9076093f94 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d419fa21-4a30-4b23-bd43-1f9076093f94', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:00,732 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:20:00,732 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:20:00,784 [287] [DEBUG] [app] Starting request: urn:request:f3c65ef1-8c15-4848-b26e-179c221dc7d7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:00,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:00,788 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:00,789 [292] [DEBUG] [app] Starting request: urn:request:c679c7f9-6063-43a8-a3cf-6935f3985e5b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:00,789 [292] [DEBUG] [app] Ending request: urn:request:c679c7f9-6063-43a8-a3cf-6935f3985e5b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c679c7f9-6063-43a8-a3cf-6935f3985e5b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:00,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:20:00,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:00,790 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:00,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:00,793 [286] [DEBUG] [app] Starting request: urn:request:ba1606d1-3ee6-4744-b192-464920f95777 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:00,793 [286] [DEBUG] [app] Ending request: urn:request:ba1606d1-3ee6-4744-b192-464920f95777 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ba1606d1-3ee6-4744-b192-464920f95777', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:00,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:00,794 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:00,794 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:00,794 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:00,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:00,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:00,811 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:00,814 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:00,817 [287] [DEBUG] [app] Ending request: urn:request:f3c65ef1-8c15-4848-b26e-179c221dc7d7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f3c65ef1-8c15-4848-b26e-179c221dc7d7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:00,817 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:00,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:01,272 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:11.530064+00:00 (in 9.999591 seconds) notificationworker stdout | 2025-11-04 09:20:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:11 GMT)" (scheduled at 2025-11-04 09:20:01.530064+00:00) notificationworker stdout | 2025-11-04 09:20:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 1, 530746), True, datetime.datetime(2025, 11, 4, 9, 20, 1, 530746), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:20:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:20:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:39.924652+00:00 (in 38.001402 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:20:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:01 GMT)" (scheduled at 2025-11-04 09:20:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:20:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:20:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 1, 923512), True, datetime.datetime(2025, 11, 4, 9, 20, 1, 923512), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:20:01,934 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:20:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:20:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:20:02,067 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:20:02,267 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:20:02,268 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:20:03,557 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:20:03,560 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:20:03,664 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:04,439 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:20:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:08.437104+00:00 (in 3.000874 seconds) gcworker stdout | 2025-11-04 09:20:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:35 GMT)" (scheduled at 2025-11-04 09:20:05.435809+00:00) gcworker stdout | 2025-11-04 09:20:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038405436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:20:05,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:20:05,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:20:05,740 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:05,782 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:20:05,796 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:20:05,796 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:20:05,806 [66] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:20:06,109 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:20:06,734 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:17.122849+00:00 (in 9.999619 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:17 GMT)" (scheduled at 2025-11-04 09:20:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 7, 123476), True, datetime.datetime(2025, 11, 4, 9, 20, 7, 123476), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:20:07,139 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,076 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,081 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,100 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,113 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,135 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,155 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,182 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,316 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:08,317 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:20:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:35.435809+00:00 (in 26.998259 seconds) gcworker stdout | 2025-11-04 09:20:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:38 GMT)" (scheduled at 2025-11-04 09:20:08.437104+00:00) gcworker stdout | 2025-11-04 09:20:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:20:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247948449, None, 1, 0]) gcworker stdout | 2025-11-04 09:20:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:38 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:20:08,528 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:08,532 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:08,542 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:20:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:20:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:57.131795+00:00 (in 47.997748 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:09 GMT)" (scheduled at 2025-11-04 09:20:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:20:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:20:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:20:09,295 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:20:11,064 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:15.535095+00:00 (in 4.004546 seconds) notificationworker stdout | 2025-11-04 09:20:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:21 GMT)" (scheduled at 2025-11-04 09:20:11.530064+00:00) notificationworker stdout | 2025-11-04 09:20:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 11, 530830), True, datetime.datetime(2025, 11, 4, 9, 20, 11, 530830), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:20:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:20:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:12.222828+00:00 (in 59.999536 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:20:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:12 GMT)" (scheduled at 2025-11-04 09:20:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:20:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:20:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:20:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:20:12,507 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:20:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:20:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:29.351216+00:00 (in 14.999543 seconds) quotatotalworker stdout | 2025-11-04 09:20:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:29 GMT)" (scheduled at 2025-11-04 09:20:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:20:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:20:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:20:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:20:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:20:15,012 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:20:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:17.479080+00:00 (in 1.998886 seconds) securityworker stdout | 2025-11-04 09:20:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:45 GMT)" (scheduled at 2025-11-04 09:20:15.479769+00:00) securityworker stdout | 2025-11-04 09:20:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 15, 483952), 1, 2]) securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 15, 483952), 1, 2]) securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:21.530064+00:00 (in 5.994509 seconds) notificationworker stdout | 2025-11-04 09:20:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:15 GMT)" (scheduled at 2025-11-04 09:20:15.535095+00:00) notificationworker stdout | 2025-11-04 09:20:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:20:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:20:15,697 [285] [DEBUG] [app] Starting request: urn:request:70d5d7bb-cd45-4884-b6c6-f4ab5b7bf7f7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:15,701 [292] [DEBUG] [app] Starting request: urn:request:80221415-badc-471c-bb37-72422b789877 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:15,701 [292] [DEBUG] [app] Ending request: urn:request:80221415-badc-471c-bb37-72422b789877 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:80221415-badc-471c-bb37-72422b789877', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:20:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:15,706 [288] [DEBUG] [app] Starting request: urn:request:91ed3de3-dec9-401d-aaac-07c62bd9afbf (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:15,706 [288] [DEBUG] [app] Ending request: urn:request:91ed3de3-dec9-401d-aaac-07c62bd9afbf (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:91ed3de3-dec9-401d-aaac-07c62bd9afbf', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:15,731 [285] [DEBUG] [app] Ending request: urn:request:70d5d7bb-cd45-4884-b6c6-f4ab5b7bf7f7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:70d5d7bb-cd45-4884-b6c6-f4ab5b7bf7f7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:15,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:15,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:20:15,783 [285] [DEBUG] [app] Starting request: urn:request:adc591b8-7fe8-40ce-a03f-70f7ba18be99 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:15,785 [299] [DEBUG] [app] Starting request: urn:request:87043c82-2ef7-445b-aecc-95f1a238483e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:15,785 [299] [DEBUG] [app] Ending request: urn:request:87043c82-2ef7-445b-aecc-95f1a238483e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:87043c82-2ef7-445b-aecc-95f1a238483e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:15,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:20:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:15,789 [288] [DEBUG] [app] Starting request: urn:request:31a5cb94-19e3-46b0-a098-f581c274d6ce (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:15,790 [288] [DEBUG] [app] Ending request: urn:request:31a5cb94-19e3-46b0-a098-f581c274d6ce (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:31a5cb94-19e3-46b0-a098-f581c274d6ce', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:15,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:15,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) builder stdout | 2025-11-04 09:20:15,806 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue gunicorn-web stdout | 2025-11-04 09:20:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:15,813 [285] [DEBUG] [app] Ending request: urn:request:adc591b8-7fe8-40ce-a03f-70f7ba18be99 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:adc591b8-7fe8-40ce-a03f-70f7ba18be99', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:15,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:15,813 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.030 47 0.031) builder stdout | 2025-11-04 09:20:15,818 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:20:15,818 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:20:15,828 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:27.122849+00:00 (in 9.999618 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:27 GMT)" (scheduled at 2025-11-04 09:20:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 17, 123539), True, datetime.datetime(2025, 11, 4, 9, 20, 17, 123539), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:20:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:45.479769+00:00 (in 28.000226 seconds) securityworker stdout | 2025-11-04 09:20:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:47 GMT)" (scheduled at 2025-11-04 09:20:17.479080+00:00) securityworker stdout | 2025-11-04 09:20:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:20:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:20:17,959 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:18,615 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:20:19,926 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:20:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:20:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:51.226294+00:00 (in 29.999584 seconds) autopruneworker stdout | 2025-11-04 09:20:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:51 GMT)" (scheduled at 2025-11-04 09:20:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:20:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247961234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:20:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:20:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:20:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:20:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:20:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:32.435813+00:00 (in 11.000960 seconds) namespacegcworker stdout | 2025-11-04 09:20:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:21 GMT)" (scheduled at 2025-11-04 09:20:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:20:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:20:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 21, 435149), True, datetime.datetime(2025, 11, 4, 9, 20, 21, 435149), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:20:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:20:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:20:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:31.530064+00:00 (in 9.999585 seconds) notificationworker stdout | 2025-11-04 09:20:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:31 GMT)" (scheduled at 2025-11-04 09:20:21.530064+00:00) notificationworker stdout | 2025-11-04 09:20:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 21, 530736), True, datetime.datetime(2025, 11, 4, 9, 20, 21, 530736), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:20:22,207 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:25,057 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:25,829 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:20:25,842 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:20:25,842 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:20:25,852 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:37.122849+00:00 (in 9.999633 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:37 GMT)" (scheduled at 2025-11-04 09:20:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 27, 123475), True, datetime.datetime(2025, 11, 4, 9, 20, 27, 123475), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:27,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:20:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:20:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:57.222846+00:00 (in 29.999566 seconds) buildlogsarchiver stdout | 2025-11-04 09:20:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:57 GMT)" (scheduled at 2025-11-04 09:20:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:20:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 20, 27, 223544), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:20:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:20:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:20:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:20:27,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:20:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:20:27,289 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:20:27,998 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:20:28,654 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:20:29,034 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:20:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:20:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:44.351216+00:00 (in 14.999498 seconds) quotatotalworker stdout | 2025-11-04 09:20:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:44 GMT)" (scheduled at 2025-11-04 09:20:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:20:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:20:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:20:29,368 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:20:29,369 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:20:29,381 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:29,608 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:29,766 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:29,782 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:29,882 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:30,697 [287] [DEBUG] [app] Starting request: urn:request:b8cce1d7-ebc2-4276-9b1a-575a78550f45 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:30,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:30,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:30,701 [292] [DEBUG] [app] Starting request: urn:request:85d25be5-ca0b-42da-aef2-4ba46db9d61d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:30,701 [292] [DEBUG] [app] Ending request: urn:request:85d25be5-ca0b-42da-aef2-4ba46db9d61d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:85d25be5-ca0b-42da-aef2-4ba46db9d61d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:20:30,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:30,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:30,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:30,705 [285] [DEBUG] [app] Starting request: urn:request:d4c2c97d-fea1-4721-a674-dfd0c8d095f8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:30,706 [285] [DEBUG] [app] Ending request: urn:request:d4c2c97d-fea1-4721-a674-dfd0c8d095f8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d4c2c97d-fea1-4721-a674-dfd0c8d095f8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:30,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:30,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:30,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:30,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:30,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:30,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:30,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:30,730 [287] [DEBUG] [app] Ending request: urn:request:b8cce1d7-ebc2-4276-9b1a-575a78550f45 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b8cce1d7-ebc2-4276-9b1a-575a78550f45', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:30,730 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:20:30,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:20:30,784 [288] [DEBUG] [app] Starting request: urn:request:eaad6671-c5e4-47d0-824d-f45ea4f74f9a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:30,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:30,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:30,788 [292] [DEBUG] [app] Starting request: urn:request:733a5dc3-9d41-4f47-a386-e8acbe5d5582 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:30,789 [292] [DEBUG] [app] Ending request: urn:request:733a5dc3-9d41-4f47-a386-e8acbe5d5582 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:733a5dc3-9d41-4f47-a386-e8acbe5d5582', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:20:30,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:30,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:30,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:30,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:30,792 [287] [DEBUG] [app] Starting request: urn:request:ac8abdf0-3a42-4e7a-a17c-742370bdfd96 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:30,793 [287] [DEBUG] [app] Ending request: urn:request:ac8abdf0-3a42-4e7a-a17c-742370bdfd96 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ac8abdf0-3a42-4e7a-a17c-742370bdfd96', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:30,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:30,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:30,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:30,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:30,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:30,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:30,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:30,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:30,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:30,817 [288] [DEBUG] [app] Ending request: urn:request:eaad6671-c5e4-47d0-824d-f45ea4f74f9a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:eaad6671-c5e4-47d0-824d-f45ea4f74f9a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:30,817 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:20:30,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:20:31,292 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:41.530064+00:00 (in 9.999549 seconds) notificationworker stdout | 2025-11-04 09:20:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:41 GMT)" (scheduled at 2025-11-04 09:20:31.530064+00:00) notificationworker stdout | 2025-11-04 09:20:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 31, 530802), True, datetime.datetime(2025, 11, 4, 9, 20, 31, 530802), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:20:32,076 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:20:32,277 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:20:32,278 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:20:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:20:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:21.434460+00:00 (in 48.998216 seconds) namespacegcworker stdout | 2025-11-04 09:20:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:32 GMT)" (scheduled at 2025-11-04 09:20:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:20:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:20:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:20:33,566 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:20:33,569 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:20:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:20:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:48.625192+00:00 (in 15.001814 seconds) exportactionlogsworker stdout | 2025-11-04 09:20:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:33 GMT)" (scheduled at 2025-11-04 09:20:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:20:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:20:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 33, 623686), True, datetime.datetime(2025, 11, 4, 9, 20, 33, 623686), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:20:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:20:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:20:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:33 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:20:33,673 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:34,449 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:20:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:38.437104+00:00 (in 3.000857 seconds) gcworker stdout | 2025-11-04 09:20:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:05 GMT)" (scheduled at 2025-11-04 09:20:35.435809+00:00) gcworker stdout | 2025-11-04 09:20:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038435436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:20:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:20:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:20:35,749 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:35,852 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:20:35,866 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:20:35,866 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:20:35,876 [66] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:20:36,119 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:20:36,744 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:40.124445+00:00 (in 3.001165 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:47 GMT)" (scheduled at 2025-11-04 09:20:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 37, 123581), True, datetime.datetime(2025, 11, 4, 9, 20, 37, 123581), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:20:37,153 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,087 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,095 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,113 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,123 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,149 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,166 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,194 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,331 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:38,333 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:20:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:05.435809+00:00 (in 26.998291 seconds) gcworker stdout | 2025-11-04 09:20:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:08 GMT)" (scheduled at 2025-11-04 09:20:38.437104+00:00) gcworker stdout | 2025-11-04 09:20:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:20:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247978449, None, 1, 0]) gcworker stdout | 2025-11-04 09:20:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:20:38,539 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:38,542 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:38,553 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:20:39,304 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:20:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:20:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:01.922827+00:00 (in 21.997737 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:20:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:39 GMT)" (scheduled at 2025-11-04 09:20:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:20:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:20:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:20:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:47.122849+00:00 (in 6.997999 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:40 GMT)" (scheduled at 2025-11-04 09:20:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:20:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:20:41,073 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:51.530064+00:00 (in 9.999591 seconds) notificationworker stdout | 2025-11-04 09:20:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:51 GMT)" (scheduled at 2025-11-04 09:20:41.530064+00:00) notificationworker stdout | 2025-11-04 09:20:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 41, 530784), True, datetime.datetime(2025, 11, 4, 9, 20, 41, 530784), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:20:42,516 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:20:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:20:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:57.523915+00:00 (in 14.090657 seconds) repositorygcworker stdout | 2025-11-04 09:20:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:43 GMT)" (scheduled at 2025-11-04 09:20:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:20:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:20:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 43, 433514), True, datetime.datetime(2025, 11, 4, 9, 20, 43, 433514), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:20:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:20:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:20:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:20:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:20:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:59.351216+00:00 (in 14.999562 seconds) quotatotalworker stdout | 2025-11-04 09:20:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:59 GMT)" (scheduled at 2025-11-04 09:20:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:20:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:20:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:20:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:20:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:20:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:20:45,021 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:20:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:47.479080+00:00 (in 1.998901 seconds) securityworker stdout | 2025-11-04 09:20:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:15 GMT)" (scheduled at 2025-11-04 09:20:45.479769+00:00) securityworker stdout | 2025-11-04 09:20:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 45, 483728), 1, 2]) securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:45,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 45, 483728), 1, 2]) securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:20:45,697 [285] [DEBUG] [app] Starting request: urn:request:779677ec-b2e3-493c-b58a-9698de297043 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:45,701 [298] [DEBUG] [app] Starting request: urn:request:638b496e-3974-4ac6-81db-8a52363efe88 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:45,701 [298] [DEBUG] [app] Ending request: urn:request:638b496e-3974-4ac6-81db-8a52363efe88 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:638b496e-3974-4ac6-81db-8a52363efe88', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:45,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:20:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:45,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:45,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:45,705 [288] [DEBUG] [app] Starting request: urn:request:944df958-f84c-4606-aa07-12f2c998f82f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:45,706 [288] [DEBUG] [app] Ending request: urn:request:944df958-f84c-4606-aa07-12f2c998f82f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:944df958-f84c-4606-aa07-12f2c998f82f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:45,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:45,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:45,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:45,730 [285] [DEBUG] [app] Ending request: urn:request:779677ec-b2e3-493c-b58a-9698de297043 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:779677ec-b2e3-493c-b58a-9698de297043', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:45,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:20:45,784 [288] [DEBUG] [app] Starting request: urn:request:27853053-2710-465b-ab10-d1662c92997b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:20:45,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:45,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:20:45,788 [292] [DEBUG] [app] Starting request: urn:request:48d60574-89ef-4777-909b-6142e62f9f36 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:20:45,788 [292] [DEBUG] [app] Ending request: urn:request:48d60574-89ef-4777-909b-6142e62f9f36 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:48d60574-89ef-4777-909b-6142e62f9f36', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:20:45,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:45,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:45,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:20:45,792 [287] [DEBUG] [app] Starting request: urn:request:99ebe129-135d-49fa-a554-ead1b74f2ea9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:20:45,792 [287] [DEBUG] [app] Ending request: urn:request:99ebe129-135d-49fa-a554-ead1b74f2ea9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:99ebe129-135d-49fa-a554-ead1b74f2ea9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:45,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:20:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:20:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:45,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:45,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:45,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:45,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:45,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:45,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:45,817 [288] [DEBUG] [app] Ending request: urn:request:27853053-2710-465b-ab10-d1662c92997b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:27853053-2710-465b-ab10-d1662c92997b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:45,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:20:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:20:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) builder stdout | 2025-11-04 09:20:45,876 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:20:45,889 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:20:45,889 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:20:45,899 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:57.122849+00:00 (in 9.999574 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:57 GMT)" (scheduled at 2025-11-04 09:20:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 47, 123570), True, datetime.datetime(2025, 11, 4, 9, 20, 47, 123570), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:20:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:15.479769+00:00 (in 28.000209 seconds) securityworker stdout | 2025-11-04 09:20:47,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:17 GMT)" (scheduled at 2025-11-04 09:20:47.479080+00:00) securityworker stdout | 2025-11-04 09:20:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 47, 483544), 1, 2]) securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 47, 483544), 1, 2]) securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:20:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:20:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:20:47,967 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:48,624 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:20:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:20:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:33.622985+00:00 (in 44.997351 seconds) exportactionlogsworker stdout | 2025-11-04 09:20:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:48 GMT)" (scheduled at 2025-11-04 09:20:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:20:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:20:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:48 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:20:49,934 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:20:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:20:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:21.226294+00:00 (in 29.999543 seconds) autopruneworker stdout | 2025-11-04 09:20:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:21 GMT)" (scheduled at 2025-11-04 09:20:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:20:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762247991235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:20:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:20:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:20:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:01.530064+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 09:20:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:01 GMT)" (scheduled at 2025-11-04 09:20:51.530064+00:00) notificationworker stdout | 2025-11-04 09:20:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 51, 530737), True, datetime.datetime(2025, 11, 4, 9, 20, 51, 530737), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:20:52,216 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:55,065 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:55,900 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:20:55,913 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:20:55,913 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:20:55,924 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:07.122849+00:00 (in 9.999568 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:07 GMT)" (scheduled at 2025-11-04 09:20:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 57, 123543), True, datetime.datetime(2025, 11, 4, 9, 20, 57, 123543), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:20:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:09.133645+00:00 (in 12.001443 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:57 GMT)" (scheduled at 2025-11-04 09:20:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:20:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:20:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:20:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:20:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:20:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:27.222846+00:00 (in 29.999584 seconds) buildlogsarchiver stdout | 2025-11-04 09:20:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:27 GMT)" (scheduled at 2025-11-04 09:20:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:20:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 20, 57, 223449), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:20:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:20:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:20:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:20:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:20:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:20:57,298 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:20:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:20:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:43.432867+00:00 (in 45.908525 seconds) repositorygcworker stdout | 2025-11-04 09:20:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:57 GMT)" (scheduled at 2025-11-04 09:20:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:20:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:20:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:20:58,007 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:20:58,663 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:20:59,042 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:20:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:20:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:14.351216+00:00 (in 14.999561 seconds) quotatotalworker stdout | 2025-11-04 09:20:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:14 GMT)" (scheduled at 2025-11-04 09:20:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:20:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:20:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:20:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:20:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:20:59,396 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:59,625 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:59,778 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:59,795 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:59,896 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:00,696 [288] [DEBUG] [app] Starting request: urn:request:25149252-209a-494b-ae88-16629316d23d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:00,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:00,700 [292] [DEBUG] [app] Starting request: urn:request:fff2dc9a-2339-4b27-91f6-e652ccf20f30 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:00,701 [292] [DEBUG] [app] Ending request: urn:request:fff2dc9a-2339-4b27-91f6-e652ccf20f30 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fff2dc9a-2339-4b27-91f6-e652ccf20f30', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:00,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:00,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:00,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:00,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:00,705 [285] [DEBUG] [app] Starting request: urn:request:3ec7eab3-70db-4bf8-bed8-ec095dfb0a25 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:00,705 [285] [DEBUG] [app] Ending request: urn:request:3ec7eab3-70db-4bf8-bed8-ec095dfb0a25 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3ec7eab3-70db-4bf8-bed8-ec095dfb0a25', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:00,705 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:00,705 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:00,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:00,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:00,730 [288] [DEBUG] [app] Ending request: urn:request:25149252-209a-494b-ae88-16629316d23d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:25149252-209a-494b-ae88-16629316d23d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:00,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:00,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:21:00,784 [288] [DEBUG] [app] Starting request: urn:request:1d472337-3a1e-4ac1-9f93-b536b735cdb6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:00,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:00,785 [292] [DEBUG] [app] Starting request: urn:request:74af1c07-5cf5-4ed9-9720-db4925833669 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:00,785 [292] [DEBUG] [app] Ending request: urn:request:74af1c07-5cf5-4ed9-9720-db4925833669 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:74af1c07-5cf5-4ed9-9720-db4925833669', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:00,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:00,786 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:00,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:00,789 [286] [DEBUG] [app] Starting request: urn:request:1a4d7391-cad5-4985-8030-b1811cb48c01 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:00,790 [286] [DEBUG] [app] Ending request: urn:request:1a4d7391-cad5-4985-8030-b1811cb48c01 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1a4d7391-cad5-4985-8030-b1811cb48c01', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:00,790 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:00,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:00,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:00,790 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:00,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:00,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:00,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:00,807 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:00,810 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:00,813 [288] [DEBUG] [app] Ending request: urn:request:1d472337-3a1e-4ac1-9f93-b536b735cdb6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1d472337-3a1e-4ac1-9f93-b536b735cdb6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:00,813 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:00,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:01,302 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:11.530064+00:00 (in 9.999582 seconds) notificationworker stdout | 2025-11-04 09:21:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:11 GMT)" (scheduled at 2025-11-04 09:21:01.530064+00:00) notificationworker stdout | 2025-11-04 09:21:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 1, 530758), True, datetime.datetime(2025, 11, 4, 9, 21, 1, 530758), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:21:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:21:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:39.924652+00:00 (in 38.001420 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:21:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:01 GMT)" (scheduled at 2025-11-04 09:21:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:21:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:21:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 1, 923465), True, datetime.datetime(2025, 11, 4, 9, 21, 1, 923465), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:21:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:21:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:21:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:21:02,085 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:21:02,286 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:21:02,287 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:21:03,574 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:21:03,577 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:21:03,682 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:04,458 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:21:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:08.437104+00:00 (in 3.000900 seconds) gcworker stdout | 2025-11-04 09:21:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:35 GMT)" (scheduled at 2025-11-04 09:21:05.435809+00:00) gcworker stdout | 2025-11-04 09:21:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038465436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:21:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:21:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:21:05,758 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:05,924 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:21:05,937 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:21:05,937 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:21:05,948 [66] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:21:06,127 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:21:06,754 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:17.122849+00:00 (in 9.999586 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:17 GMT)" (scheduled at 2025-11-04 09:21:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 7, 123519), True, datetime.datetime(2025, 11, 4, 9, 21, 7, 123519), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:21:07,162 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,097 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,107 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,125 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,133 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,163 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,177 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,206 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,346 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:08,348 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:21:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:35.435809+00:00 (in 26.998262 seconds) gcworker stdout | 2025-11-04 09:21:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:38 GMT)" (scheduled at 2025-11-04 09:21:08.437104+00:00) gcworker stdout | 2025-11-04 09:21:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:21:08,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248008449, None, 1, 0]) gcworker stdout | 2025-11-04 09:21:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:38 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:21:08,550 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:08,553 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:08,563 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:21:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:21:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:57.131795+00:00 (in 47.997722 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:09 GMT)" (scheduled at 2025-11-04 09:21:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:21:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:21:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:21:09,312 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:21:11,082 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:15.535095+00:00 (in 4.004544 seconds) notificationworker stdout | 2025-11-04 09:21:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:21 GMT)" (scheduled at 2025-11-04 09:21:11.530064+00:00) notificationworker stdout | 2025-11-04 09:21:11,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 11, 531182), True, datetime.datetime(2025, 11, 4, 9, 21, 11, 531182), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:21:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:21:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:12.222828+00:00 (in 59.999545 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:21:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:12 GMT)" (scheduled at 2025-11-04 09:21:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:21:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:21:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:21:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:21:12,525 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:21:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:21:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:29.351216+00:00 (in 14.999589 seconds) quotatotalworker stdout | 2025-11-04 09:21:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:29 GMT)" (scheduled at 2025-11-04 09:21:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:21:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:21:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:21:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:21:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:21:15,029 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:21:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:17.479080+00:00 (in 1.998884 seconds) securityworker stdout | 2025-11-04 09:21:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:45 GMT)" (scheduled at 2025-11-04 09:21:15.479769+00:00) securityworker stdout | 2025-11-04 09:21:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:15,496 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:15,500 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:15,500 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 15, 484096), 1, 2]) securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 15, 484096), 1, 2]) securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:21:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:21.530064+00:00 (in 5.994587 seconds) notificationworker stdout | 2025-11-04 09:21:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:15 GMT)" (scheduled at 2025-11-04 09:21:15.535095+00:00) notificationworker stdout | 2025-11-04 09:21:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:21:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:21:15,697 [287] [DEBUG] [app] Starting request: urn:request:07362bf1-7847-47ba-b593-b5aee20e1bfd (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:15,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:15,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:15,701 [298] [DEBUG] [app] Starting request: urn:request:cd86c7d5-b98a-404b-b4b4-5b9439a964f1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:15,702 [298] [DEBUG] [app] Ending request: urn:request:cd86c7d5-b98a-404b-b4b4-5b9439a964f1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cd86c7d5-b98a-404b-b4b4-5b9439a964f1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:15,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:15,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:15,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:15,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:15,706 [288] [DEBUG] [app] Starting request: urn:request:a9429645-56ad-459d-8610-ff0787ee2507 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:15,706 [288] [DEBUG] [app] Ending request: urn:request:a9429645-56ad-459d-8610-ff0787ee2507 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a9429645-56ad-459d-8610-ff0787ee2507', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:15,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:15,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:15,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:15,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:15,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:15,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:15,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:15,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:15,730 [287] [DEBUG] [app] Ending request: urn:request:07362bf1-7847-47ba-b593-b5aee20e1bfd (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:07362bf1-7847-47ba-b593-b5aee20e1bfd', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:15,731 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:21:15,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:21:15,784 [287] [DEBUG] [app] Starting request: urn:request:3c329fe5-f9ee-4a19-a8aa-f16d875a0a46 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:15,785 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:15,785 [298] [DEBUG] [app] Starting request: urn:request:fa26f2ce-2d48-4a58-b5aa-f02500f59f26 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:15,786 [298] [DEBUG] [app] Ending request: urn:request:fa26f2ce-2d48-4a58-b5aa-f02500f59f26 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fa26f2ce-2d48-4a58-b5aa-f02500f59f26', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:15,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:15,786 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:15,787 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:15,789 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:15,790 [288] [DEBUG] [app] Starting request: urn:request:4563ea51-8a8f-4418-9f66-4816ad148778 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:15,790 [288] [DEBUG] [app] Ending request: urn:request:4563ea51-8a8f-4418-9f66-4816ad148778 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4563ea51-8a8f-4418-9f66-4816ad148778', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:15,790 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:15,791 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:15,791 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:15,791 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:15,798 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:15,798 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:15,808 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:15,812 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:15,815 [287] [DEBUG] [app] Ending request: urn:request:3c329fe5-f9ee-4a19-a8aa-f16d875a0a46 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3c329fe5-f9ee-4a19-a8aa-f16d875a0a46', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:15,815 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:15,815 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) builder stdout | 2025-11-04 09:21:15,948 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:21:15,961 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:21:15,961 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:21:15,971 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:27.122849+00:00 (in 9.999616 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:27 GMT)" (scheduled at 2025-11-04 09:21:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 17, 123509), True, datetime.datetime(2025, 11, 4, 9, 21, 17, 123509), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:17,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:21:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:45.479769+00:00 (in 28.000214 seconds) securityworker stdout | 2025-11-04 09:21:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:47 GMT)" (scheduled at 2025-11-04 09:21:17.479080+00:00) securityworker stdout | 2025-11-04 09:21:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:21:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:21:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:21:17,976 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:18,633 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:21:19,943 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:21:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:21:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:51.226294+00:00 (in 29.999558 seconds) autopruneworker stdout | 2025-11-04 09:21:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:51 GMT)" (scheduled at 2025-11-04 09:21:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:21:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248021234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:21:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:21:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:21:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:21:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:21:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:32.435813+00:00 (in 11.000958 seconds) namespacegcworker stdout | 2025-11-04 09:21:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:21 GMT)" (scheduled at 2025-11-04 09:21:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:21:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:21:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 21, 435138), True, datetime.datetime(2025, 11, 4, 9, 21, 21, 435138), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:21:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:21:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:21:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:31.530064+00:00 (in 9.999623 seconds) notificationworker stdout | 2025-11-04 09:21:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:31 GMT)" (scheduled at 2025-11-04 09:21:21.530064+00:00) notificationworker stdout | 2025-11-04 09:21:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 21, 530721), True, datetime.datetime(2025, 11, 4, 9, 21, 21, 530721), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:21:22,225 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:25,074 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:25,972 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:21:25,985 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:21:25,985 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:21:25,994 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:37.122849+00:00 (in 9.999568 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:37 GMT)" (scheduled at 2025-11-04 09:21:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 27, 123597), True, datetime.datetime(2025, 11, 4, 9, 21, 27, 123597), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:21:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:21:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:57.222846+00:00 (in 29.999601 seconds) buildlogsarchiver stdout | 2025-11-04 09:21:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:57 GMT)" (scheduled at 2025-11-04 09:21:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:21:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 21, 27, 223455), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:21:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:21:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:21:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:21:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:21:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:21:27,308 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:21:28,016 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:21:28,672 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:21:29,051 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:21:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:21:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:44.351216+00:00 (in 14.999585 seconds) quotatotalworker stdout | 2025-11-04 09:21:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:44 GMT)" (scheduled at 2025-11-04 09:21:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:21:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:21:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:21:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:21:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:21:29,407 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:29,637 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:29,793 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:29,809 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:29,910 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:30,697 [288] [DEBUG] [app] Starting request: urn:request:5e17fbc5-c20b-4129-8377-676673c04594 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:30,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:30,702 [292] [DEBUG] [app] Starting request: urn:request:0de06aa7-dc36-41bc-916c-d398967a72e9 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:30,702 [292] [DEBUG] [app] Ending request: urn:request:0de06aa7-dc36-41bc-916c-d398967a72e9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0de06aa7-dc36-41bc-916c-d398967a72e9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:30,703 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:30,706 [285] [DEBUG] [app] Starting request: urn:request:0180518a-74e9-4287-831d-6ec8716593ca (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:30,706 [285] [DEBUG] [app] Ending request: urn:request:0180518a-74e9-4287-831d-6ec8716593ca (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0180518a-74e9-4287-831d-6ec8716593ca', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:30,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:30,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:30,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:30,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:30,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:30,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:30,731 [288] [DEBUG] [app] Ending request: urn:request:5e17fbc5-c20b-4129-8377-676673c04594 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5e17fbc5-c20b-4129-8377-676673c04594', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:30,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:21:30,783 [285] [DEBUG] [app] Starting request: urn:request:e3095b90-7f39-4d03-a005-1f18105d2633 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:30,788 [292] [DEBUG] [app] Starting request: urn:request:158cc1c0-b343-4d51-a8f8-7b6a8b4106fe (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:30,788 [292] [DEBUG] [app] Ending request: urn:request:158cc1c0-b343-4d51-a8f8-7b6a8b4106fe (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:158cc1c0-b343-4d51-a8f8-7b6a8b4106fe', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:30,792 [287] [DEBUG] [app] Starting request: urn:request:02bd7da6-5d43-46ad-9f44-f490d9750d45 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:30,792 [287] [DEBUG] [app] Ending request: urn:request:02bd7da6-5d43-46ad-9f44-f490d9750d45 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:02bd7da6-5d43-46ad-9f44-f490d9750d45', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:30,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:30,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:30,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:30,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:30,816 [285] [DEBUG] [app] Ending request: urn:request:e3095b90-7f39-4d03-a005-1f18105d2633 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e3095b90-7f39-4d03-a005-1f18105d2633', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:30,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:31,311 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:41.530064+00:00 (in 9.999553 seconds) notificationworker stdout | 2025-11-04 09:21:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:41 GMT)" (scheduled at 2025-11-04 09:21:31.530064+00:00) notificationworker stdout | 2025-11-04 09:21:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 31, 530829), True, datetime.datetime(2025, 11, 4, 9, 21, 31, 530829), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:21:32,093 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:21:32,295 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:21:32,296 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:21:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:21:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:21.434460+00:00 (in 48.998282 seconds) namespacegcworker stdout | 2025-11-04 09:21:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:32 GMT)" (scheduled at 2025-11-04 09:21:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:21:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:21:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:21:33,584 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:21:33,586 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:21:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:21:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:48.625192+00:00 (in 15.001801 seconds) exportactionlogsworker stdout | 2025-11-04 09:21:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:33 GMT)" (scheduled at 2025-11-04 09:21:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:21:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:21:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 33, 623718), True, datetime.datetime(2025, 11, 4, 9, 21, 33, 623718), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:21:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:21:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:21:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:33 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:21:33,690 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:34,467 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:21:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:38.437104+00:00 (in 3.000898 seconds) gcworker stdout | 2025-11-04 09:21:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:05 GMT)" (scheduled at 2025-11-04 09:21:35.435809+00:00) gcworker stdout | 2025-11-04 09:21:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038495436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:21:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:21:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:21:35,766 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:35,995 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:21:36,009 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:21:36,009 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:21:36,019 [66] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:21:36,136 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:21:36,763 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:40.124445+00:00 (in 3.001164 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:47 GMT)" (scheduled at 2025-11-04 09:21:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 37, 123573), True, datetime.datetime(2025, 11, 4, 9, 21, 37, 123573), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:21:37,171 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,112 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,119 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,137 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,147 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,178 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,187 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,217 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,360 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:38,363 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:21:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:05.435809+00:00 (in 26.998286 seconds) gcworker stdout | 2025-11-04 09:21:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:08 GMT)" (scheduled at 2025-11-04 09:21:38.437104+00:00) gcworker stdout | 2025-11-04 09:21:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:21:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248038449, None, 1, 0]) gcworker stdout | 2025-11-04 09:21:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:21:38,561 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:38,564 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:38,574 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:21:39,321 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:21:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:21:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:01.922827+00:00 (in 21.997764 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:21:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:39 GMT)" (scheduled at 2025-11-04 09:21:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:21:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:21:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:21:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:47.122849+00:00 (in 6.998002 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:40 GMT)" (scheduled at 2025-11-04 09:21:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:21:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:21:41,091 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:51.530064+00:00 (in 9.999551 seconds) notificationworker stdout | 2025-11-04 09:21:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:51 GMT)" (scheduled at 2025-11-04 09:21:41.530064+00:00) notificationworker stdout | 2025-11-04 09:21:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 41, 530790), True, datetime.datetime(2025, 11, 4, 9, 21, 41, 530790), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:21:42,534 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:21:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:21:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:57.523915+00:00 (in 14.090634 seconds) repositorygcworker stdout | 2025-11-04 09:21:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:43 GMT)" (scheduled at 2025-11-04 09:21:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:21:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:21:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 43, 433541), True, datetime.datetime(2025, 11, 4, 9, 21, 43, 433541), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:21:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:21:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:21:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:21:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:21:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:59.351216+00:00 (in 14.999590 seconds) quotatotalworker stdout | 2025-11-04 09:21:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:59 GMT)" (scheduled at 2025-11-04 09:21:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:21:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:21:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:21:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:21:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:21:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:21:45,038 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:21:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:47.479080+00:00 (in 1.998861 seconds) securityworker stdout | 2025-11-04 09:21:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:15 GMT)" (scheduled at 2025-11-04 09:21:45.479769+00:00) securityworker stdout | 2025-11-04 09:21:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:45,481 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:45,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:45,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:45,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:45,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:45,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 45, 484212), 1, 2]) securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 45, 484212), 1, 2]) securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:21:45,697 [285] [DEBUG] [app] Starting request: urn:request:c437fb6b-c545-4457-a0cd-b1c48e988c2a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:45,701 [298] [DEBUG] [app] Starting request: urn:request:46ec9975-eda4-40a4-8485-bad2f6e628ea (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:45,702 [298] [DEBUG] [app] Ending request: urn:request:46ec9975-eda4-40a4-8485-bad2f6e628ea (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:46ec9975-eda4-40a4-8485-bad2f6e628ea', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:45,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:45,706 [287] [DEBUG] [app] Starting request: urn:request:a684ba25-9ab6-42af-b464-fea6f42c7748 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:45,706 [287] [DEBUG] [app] Ending request: urn:request:a684ba25-9ab6-42af-b464-fea6f42c7748 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a684ba25-9ab6-42af-b464-fea6f42c7748', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:45,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:21:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:45,730 [285] [DEBUG] [app] Ending request: urn:request:c437fb6b-c545-4457-a0cd-b1c48e988c2a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c437fb6b-c545-4457-a0cd-b1c48e988c2a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:45,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:21:45,784 [288] [DEBUG] [app] Starting request: urn:request:52558cf0-604c-43d2-bf1f-b26071af2135 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:21:45,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:45,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:21:45,788 [292] [DEBUG] [app] Starting request: urn:request:f960fad9-7596-49a8-b1c3-2a6142c58639 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:21:45,789 [292] [DEBUG] [app] Ending request: urn:request:f960fad9-7596-49a8-b1c3-2a6142c58639 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f960fad9-7596-49a8-b1c3-2a6142c58639', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:21:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:45,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:45,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:45,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:21:45,793 [285] [DEBUG] [app] Starting request: urn:request:68067831-3aca-464d-96db-24b8d7bc5e0e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:21:45,793 [285] [DEBUG] [app] Ending request: urn:request:68067831-3aca-464d-96db-24b8d7bc5e0e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:68067831-3aca-464d-96db-24b8d7bc5e0e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:21:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:21:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:21:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:45,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:45,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:45,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:45,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:45,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:45,817 [288] [DEBUG] [app] Ending request: urn:request:52558cf0-604c-43d2-bf1f-b26071af2135 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:52558cf0-604c-43d2-bf1f-b26071af2135', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:45,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:21:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:21:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) builder stdout | 2025-11-04 09:21:46,019 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:21:46,033 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:21:46,033 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:21:46,043 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:57.122849+00:00 (in 9.999629 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:57 GMT)" (scheduled at 2025-11-04 09:21:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:47,123 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 47, 123432), True, datetime.datetime(2025, 11, 4, 9, 21, 47, 123432), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:21:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:15.479769+00:00 (in 28.000226 seconds) securityworker stdout | 2025-11-04 09:21:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:17 GMT)" (scheduled at 2025-11-04 09:21:47.479080+00:00) securityworker stdout | 2025-11-04 09:21:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:21:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:21:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:47,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 47, 483295), 1, 2]) securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:47,508 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 47, 483295), 1, 2]) securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:21:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:21:47,984 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:21:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:21:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:58.624878+00:00 (in 9.999237 seconds) exportactionlogsworker stdout | 2025-11-04 09:21:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:48 GMT)" (scheduled at 2025-11-04 09:21:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:21:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:21:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:21:48,642 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:21:49,952 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:21:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:21:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:21.226294+00:00 (in 29.999566 seconds) autopruneworker stdout | 2025-11-04 09:21:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:21 GMT)" (scheduled at 2025-11-04 09:21:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:21:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248051234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:21:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:21:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:21:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:01.530064+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 09:21:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:01 GMT)" (scheduled at 2025-11-04 09:21:51.530064+00:00) notificationworker stdout | 2025-11-04 09:21:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 51, 530795), True, datetime.datetime(2025, 11, 4, 9, 21, 51, 530795), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:21:52,234 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:55,084 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:56,043 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:21:56,056 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:21:56,057 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:21:56,067 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:07.122849+00:00 (in 9.999581 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:07 GMT)" (scheduled at 2025-11-04 09:21:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 57, 123531), True, datetime.datetime(2025, 11, 4, 9, 21, 57, 123531), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:21:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:09.133645+00:00 (in 12.001445 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:57 GMT)" (scheduled at 2025-11-04 09:21:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:21:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:21:57,144 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:21:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:21:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:21:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:27.222846+00:00 (in 29.999384 seconds) buildlogsarchiver stdout | 2025-11-04 09:21:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:27 GMT)" (scheduled at 2025-11-04 09:21:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:21:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 21, 57, 223750), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:21:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:21:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:21:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:21:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:21:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:21:57,317 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:21:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:21:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:43.432867+00:00 (in 45.908507 seconds) repositorygcworker stdout | 2025-11-04 09:21:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:57 GMT)" (scheduled at 2025-11-04 09:21:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:21:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:21:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:21:58,025 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:21:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:21:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:33.622985+00:00 (in 34.997625 seconds) exportactionlogsworker stdout | 2025-11-04 09:21:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:26:58 GMT)" (scheduled at 2025-11-04 09:21:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:21:58,626 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 21, 58, 625702), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:21:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 21, 58, 625702), True, datetime.datetime(2025, 11, 4, 9, 21, 58, 625702), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:21:58,651 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 21, 58, 625702), True, datetime.datetime(2025, 11, 4, 9, 21, 58, 625702), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 21, 58, 625702), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:21:58,654 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:21:58,654 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:26:58 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:21:58,681 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:21:59,060 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:21:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:21:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:14.351216+00:00 (in 14.999591 seconds) quotatotalworker stdout | 2025-11-04 09:21:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:14 GMT)" (scheduled at 2025-11-04 09:21:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:21:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:21:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:21:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:21:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:21:59,431 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:59,648 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:59,813 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:59,822 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:59,924 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:00,696 [288] [DEBUG] [app] Starting request: urn:request:dfee4606-97ec-4d17-9730-8ad34a2a571b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:00,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:00,699 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:00,700 [292] [DEBUG] [app] Starting request: urn:request:80753822-23ed-4cb7-9026-5c2d88621a56 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:00,701 [292] [DEBUG] [app] Ending request: urn:request:80753822-23ed-4cb7-9026-5c2d88621a56 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:80753822-23ed-4cb7-9026-5c2d88621a56', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:00,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:22:00,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:00,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:00,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:00,705 [287] [DEBUG] [app] Starting request: urn:request:05f1eebf-8029-478b-9dd1-47d31024ab92 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:00,705 [287] [DEBUG] [app] Ending request: urn:request:05f1eebf-8029-478b-9dd1-47d31024ab92 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:05f1eebf-8029-478b-9dd1-47d31024ab92', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:00,705 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:00,705 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:00,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:00,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:00,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:00,730 [288] [DEBUG] [app] Ending request: urn:request:dfee4606-97ec-4d17-9730-8ad34a2a571b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:dfee4606-97ec-4d17-9730-8ad34a2a571b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:00,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:00,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:22:00,784 [288] [DEBUG] [app] Starting request: urn:request:4bdca7f9-29b6-441a-9070-3a5a488402e2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:00,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:00,785 [298] [DEBUG] [app] Starting request: urn:request:715fe7b4-1f07-4076-8a5d-82112395adde (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:00,786 [298] [DEBUG] [app] Ending request: urn:request:715fe7b4-1f07-4076-8a5d-82112395adde (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:715fe7b4-1f07-4076-8a5d-82112395adde', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:00,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:22:00,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:00,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:00,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:00,790 [287] [DEBUG] [app] Starting request: urn:request:9b6e4917-809e-4fc9-bd9e-44b553417dcf (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:00,790 [287] [DEBUG] [app] Ending request: urn:request:9b6e4917-809e-4fc9-bd9e-44b553417dcf (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9b6e4917-809e-4fc9-bd9e-44b553417dcf', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:00,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:22:00,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:00,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:00,791 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:00,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:00,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:00,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:00,808 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:00,812 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:00,815 [288] [DEBUG] [app] Ending request: urn:request:4bdca7f9-29b6-441a-9070-3a5a488402e2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4bdca7f9-29b6-441a-9070-3a5a488402e2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:00,815 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.033) gunicorn-web stdout | 2025-11-04 09:22:00,815 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:22:01,319 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:11.530064+00:00 (in 9.999550 seconds) notificationworker stdout | 2025-11-04 09:22:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:11 GMT)" (scheduled at 2025-11-04 09:22:01.530064+00:00) notificationworker stdout | 2025-11-04 09:22:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 1, 530829), True, datetime.datetime(2025, 11, 4, 9, 22, 1, 530829), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:01,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:22:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:22:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:21.924304+00:00 (in 20.001091 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:22:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:01 GMT)" (scheduled at 2025-11-04 09:22:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:22:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:22:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 1, 923481), True, datetime.datetime(2025, 11, 4, 9, 22, 1, 923481), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:22:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:22:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:22:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:22:02,114 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:22:02,304 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:22:02,305 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:22:03,592 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:22:03,594 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:22:03,699 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:04,477 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:22:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:08.437104+00:00 (in 3.000886 seconds) gcworker stdout | 2025-11-04 09:22:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:35 GMT)" (scheduled at 2025-11-04 09:22:05.435809+00:00) gcworker stdout | 2025-11-04 09:22:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038525436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:22:05,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:22:05,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:22:05,775 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:06,067 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:22:06,081 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:22:06,081 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:22:06,091 [66] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:22:06,145 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:22:06,772 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:17.122849+00:00 (in 9.999564 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:17 GMT)" (scheduled at 2025-11-04 09:22:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 7, 123920), True, datetime.datetime(2025, 11, 4, 9, 22, 7, 123920), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:22:07,180 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,127 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,131 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,147 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,161 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,190 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,200 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,227 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,372 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:08,377 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:22:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:35.435809+00:00 (in 26.998338 seconds) gcworker stdout | 2025-11-04 09:22:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:38 GMT)" (scheduled at 2025-11-04 09:22:08.437104+00:00) gcworker stdout | 2025-11-04 09:22:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:22:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248068449, None, 1, 0]) gcworker stdout | 2025-11-04 09:22:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:38 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:22:08,571 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:08,575 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:08,584 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:22:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:22:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:57.131795+00:00 (in 47.997773 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:09 GMT)" (scheduled at 2025-11-04 09:22:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:22:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:22:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:22:09,331 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:22:11,100 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:15.535095+00:00 (in 4.004638 seconds) notificationworker stdout | 2025-11-04 09:22:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:21 GMT)" (scheduled at 2025-11-04 09:22:11.530064+00:00) notificationworker stdout | 2025-11-04 09:22:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 11, 530786), True, datetime.datetime(2025, 11, 4, 9, 22, 11, 530786), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:22:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:22:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:12.222828+00:00 (in 59.999561 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:22:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:12 GMT)" (scheduled at 2025-11-04 09:22:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:22:12,224 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:22:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:22:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:22:12,548 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:22:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:22:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:29.351216+00:00 (in 14.999646 seconds) quotatotalworker stdout | 2025-11-04 09:22:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:29 GMT)" (scheduled at 2025-11-04 09:22:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:22:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:22:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:22:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:22:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:22:15,046 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:22:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:17.479080+00:00 (in 1.998890 seconds) securityworker stdout | 2025-11-04 09:22:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:45 GMT)" (scheduled at 2025-11-04 09:22:15.479769+00:00) securityworker stdout | 2025-11-04 09:22:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 15, 483984), 1, 2]) securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 15, 483984), 1, 2]) securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:21.530064+00:00 (in 5.994518 seconds) notificationworker stdout | 2025-11-04 09:22:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:15 GMT)" (scheduled at 2025-11-04 09:22:15.535095+00:00) notificationworker stdout | 2025-11-04 09:22:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:22:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:22:15,697 [285] [DEBUG] [app] Starting request: urn:request:3094b6e3-0986-4fe0-bb56-7e3e6e949f28 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:15,701 [292] [DEBUG] [app] Starting request: urn:request:5bc160c5-f7da-445c-bd84-12b508a977a7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:15,702 [292] [DEBUG] [app] Ending request: urn:request:5bc160c5-f7da-445c-bd84-12b508a977a7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5bc160c5-f7da-445c-bd84-12b508a977a7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:22:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:15,706 [288] [DEBUG] [app] Starting request: urn:request:797b72d4-3a04-43c1-9ec5-f663a18ec895 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:15,706 [288] [DEBUG] [app] Ending request: urn:request:797b72d4-3a04-43c1-9ec5-f663a18ec895 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:797b72d4-3a04-43c1-9ec5-f663a18ec895', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:15,730 [285] [DEBUG] [app] Ending request: urn:request:3094b6e3-0986-4fe0-bb56-7e3e6e949f28 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3094b6e3-0986-4fe0-bb56-7e3e6e949f28', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:15,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:22:15,784 [285] [DEBUG] [app] Starting request: urn:request:9f830651-a9be-43ef-9997-f4a83dec8b79 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:15,785 [298] [DEBUG] [app] Starting request: urn:request:89ba3889-5646-4734-8f3c-1476f2769a53 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:15,786 [298] [DEBUG] [app] Ending request: urn:request:89ba3889-5646-4734-8f3c-1476f2769a53 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:89ba3889-5646-4734-8f3c-1476f2769a53', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:22:15,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:15,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:15,790 [288] [DEBUG] [app] Starting request: urn:request:0ef35605-cf29-46cc-93d6-843eb45a2c65 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:15,790 [288] [DEBUG] [app] Ending request: urn:request:0ef35605-cf29-46cc-93d6-843eb45a2c65 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0ef35605-cf29-46cc-93d6-843eb45a2c65', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:15,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:15,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:15,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:15,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:15,812 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:15,815 [285] [DEBUG] [app] Ending request: urn:request:9f830651-a9be-43ef-9997-f4a83dec8b79 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9f830651-a9be-43ef-9997-f4a83dec8b79', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:15,815 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:15,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.033) builder stdout | 2025-11-04 09:22:16,091 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:22:16,103 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:22:16,103 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:22:16,113 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:27.122849+00:00 (in 9.999613 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:27 GMT)" (scheduled at 2025-11-04 09:22:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 17, 123547), True, datetime.datetime(2025, 11, 4, 9, 22, 17, 123547), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:22:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:45.479769+00:00 (in 28.000301 seconds) securityworker stdout | 2025-11-04 09:22:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:47 GMT)" (scheduled at 2025-11-04 09:22:17.479080+00:00) securityworker stdout | 2025-11-04 09:22:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:22:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:22:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:22:17,993 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:18,651 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:22:19,961 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:22:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:22:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:51.226294+00:00 (in 29.999494 seconds) autopruneworker stdout | 2025-11-04 09:22:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:51 GMT)" (scheduled at 2025-11-04 09:22:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:22:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248081234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:22:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:22:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:22:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:22:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:22:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:32.435813+00:00 (in 11.000994 seconds) namespacegcworker stdout | 2025-11-04 09:22:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:21 GMT)" (scheduled at 2025-11-04 09:22:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:22:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:22:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 21, 435094), True, datetime.datetime(2025, 11, 4, 9, 22, 21, 435094), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:22:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:22:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:22:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:31.530064+00:00 (in 9.999580 seconds) notificationworker stdout | 2025-11-04 09:22:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:31 GMT)" (scheduled at 2025-11-04 09:22:21.530064+00:00) notificationworker stdout | 2025-11-04 09:22:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 21, 530781), True, datetime.datetime(2025, 11, 4, 9, 22, 21, 530781), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:31 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:22:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:22:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:39.924652+00:00 (in 17.999954 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:22:21,924 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:21 GMT)" (scheduled at 2025-11-04 09:22:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:22:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 22, 21, 924934), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:22:21,936 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 21, 924934), True, datetime.datetime(2025, 11, 4, 9, 22, 21, 924934), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:22:21,940 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 21, 924934), True, datetime.datetime(2025, 11, 4, 9, 22, 21, 924934), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 22, 21, 924934), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:22:21,944 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:22:21,944 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:21 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:22:22,243 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:25,093 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:26,114 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:22:26,127 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:22:26,127 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:22:26,137 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:37.122849+00:00 (in 9.999621 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:37 GMT)" (scheduled at 2025-11-04 09:22:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 27, 123508), True, datetime.datetime(2025, 11, 4, 9, 22, 27, 123508), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:27,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:27,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:27,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:22:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:22:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:57.222846+00:00 (in 29.999544 seconds) buildlogsarchiver stdout | 2025-11-04 09:22:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:57 GMT)" (scheduled at 2025-11-04 09:22:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:22:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 22, 27, 223522), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:22:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:22:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:22:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:22:27,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:22:27,248 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:22:27,326 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:22:28,033 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:22:28,689 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:22:29,069 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:22:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:22:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:44.351216+00:00 (in 14.999639 seconds) quotatotalworker stdout | 2025-11-04 09:22:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:44 GMT)" (scheduled at 2025-11-04 09:22:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:22:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:22:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:22:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:22:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:22:29,443 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:29,660 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:29,827 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:29,835 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:29,938 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:30,697 [285] [DEBUG] [app] Starting request: urn:request:d2f8bd14-b08c-471e-920d-8ad90f6bd71e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:30,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:30,701 [299] [DEBUG] [app] Starting request: urn:request:91a50bf5-6e71-4f2d-98ab-4452524794f5 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:30,702 [299] [DEBUG] [app] Ending request: urn:request:91a50bf5-6e71-4f2d-98ab-4452524794f5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:91a50bf5-6e71-4f2d-98ab-4452524794f5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:22:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:30,706 [286] [DEBUG] [app] Starting request: urn:request:50716c2c-ba08-4c35-8e74-724b4d7e9db4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:30,706 [286] [DEBUG] [app] Ending request: urn:request:50716c2c-ba08-4c35-8e74-724b4d7e9db4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:50716c2c-ba08-4c35-8e74-724b4d7e9db4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:30,706 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:30,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:30,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:30,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:30,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:30,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:30,731 [285] [DEBUG] [app] Ending request: urn:request:d2f8bd14-b08c-471e-920d-8ad90f6bd71e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d2f8bd14-b08c-471e-920d-8ad90f6bd71e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:30,731 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:22:30,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:22:30,784 [288] [DEBUG] [app] Starting request: urn:request:a8940838-fcf4-4533-88c4-479da1f0b08b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:30,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:30,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:30,789 [299] [DEBUG] [app] Starting request: urn:request:9f29cd53-18b5-4ca6-ac1c-e725015f4db1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:30,789 [299] [DEBUG] [app] Ending request: urn:request:9f29cd53-18b5-4ca6-ac1c-e725015f4db1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9f29cd53-18b5-4ca6-ac1c-e725015f4db1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:22:30,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:30,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:30,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:30,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:30,793 [285] [DEBUG] [app] Starting request: urn:request:4f8b8c43-5310-467b-8ae2-3bc505132113 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:30,793 [285] [DEBUG] [app] Ending request: urn:request:4f8b8c43-5310-467b-8ae2-3bc505132113 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4f8b8c43-5310-467b-8ae2-3bc505132113', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:30,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:30,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:30,794 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:30,794 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:30,794 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:30,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:30,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:30,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:30,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:30,817 [288] [DEBUG] [app] Ending request: urn:request:a8940838-fcf4-4533-88c4-479da1f0b08b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a8940838-fcf4-4533-88c4-479da1f0b08b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:30,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:30,818 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:31,329 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:41.530064+00:00 (in 9.999538 seconds) notificationworker stdout | 2025-11-04 09:22:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:41 GMT)" (scheduled at 2025-11-04 09:22:31.530064+00:00) notificationworker stdout | 2025-11-04 09:22:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 31, 530814), True, datetime.datetime(2025, 11, 4, 9, 22, 31, 530814), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:22:32,124 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:22:32,313 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:22:32,314 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:22:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:22:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:08.435432+00:00 (in 35.999237 seconds) namespacegcworker stdout | 2025-11-04 09:22:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:32 GMT)" (scheduled at 2025-11-04 09:22:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:22:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:22:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:22:33,601 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:22:33,603 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:22:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:22:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:48.625192+00:00 (in 15.001828 seconds) exportactionlogsworker stdout | 2025-11-04 09:22:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:33 GMT)" (scheduled at 2025-11-04 09:22:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:22:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:22:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 33, 623630), True, datetime.datetime(2025, 11, 4, 9, 22, 33, 623630), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:22:33,635 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:22:33,635 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:22:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:33 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:22:33,708 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:34,486 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:22:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:38.437104+00:00 (in 3.000880 seconds) gcworker stdout | 2025-11-04 09:22:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:05 GMT)" (scheduled at 2025-11-04 09:22:35.435809+00:00) gcworker stdout | 2025-11-04 09:22:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038555436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:22:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:22:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:22:35,795 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:36,137 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:22:36,151 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:22:36,151 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:22:36,161 [66] [DEBUG] [data.database] Disconnecting from database. servicekey stdout | 2025-11-04 09:22:36,170 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:22:36,782 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:40.124445+00:00 (in 3.001165 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:47 GMT)" (scheduled at 2025-11-04 09:22:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 37, 123580), True, datetime.datetime(2025, 11, 4, 9, 22, 37, 123580), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:22:37,189 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,140 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,141 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,158 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,187 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,202 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,213 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,238 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,385 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:38,395 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:22:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:05.435809+00:00 (in 26.998293 seconds) gcworker stdout | 2025-11-04 09:22:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:08 GMT)" (scheduled at 2025-11-04 09:22:38.437104+00:00) gcworker stdout | 2025-11-04 09:22:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:22:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248098449, None, 1, 0]) gcworker stdout | 2025-11-04 09:22:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:22:38,582 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:38,586 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:38,594 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:22:39,340 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:22:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:22:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:01.922827+00:00 (in 21.997750 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:22:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:39 GMT)" (scheduled at 2025-11-04 09:22:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:22:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:22:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:22:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:47.122849+00:00 (in 6.997981 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:40 GMT)" (scheduled at 2025-11-04 09:22:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:22:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:22:41,109 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:51.530064+00:00 (in 9.999548 seconds) notificationworker stdout | 2025-11-04 09:22:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:51 GMT)" (scheduled at 2025-11-04 09:22:41.530064+00:00) notificationworker stdout | 2025-11-04 09:22:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 41, 530801), True, datetime.datetime(2025, 11, 4, 9, 22, 41, 530801), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:22:42,557 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:22:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:22:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:57.523915+00:00 (in 14.090667 seconds) repositorygcworker stdout | 2025-11-04 09:22:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:43 GMT)" (scheduled at 2025-11-04 09:22:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:22:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:22:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 43, 433520), True, datetime.datetime(2025, 11, 4, 9, 22, 43, 433520), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:22:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:22:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:22:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:22:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:22:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:59.351216+00:00 (in 14.999597 seconds) quotatotalworker stdout | 2025-11-04 09:22:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:59 GMT)" (scheduled at 2025-11-04 09:22:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:22:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:22:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:22:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:22:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:22:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:22:45,055 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:22:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:47.479080+00:00 (in 1.998879 seconds) securityworker stdout | 2025-11-04 09:22:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:15 GMT)" (scheduled at 2025-11-04 09:22:45.479769+00:00) securityworker stdout | 2025-11-04 09:22:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:22:45,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:45,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:45,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:45,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:45,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 45, 484670), 1, 2]) securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:45,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 45, 484670), 1, 2]) securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:22:45,697 [288] [DEBUG] [app] Starting request: urn:request:53b504a6-4233-40d4-b369-5f469aef2446 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:45,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:45,701 [298] [DEBUG] [app] Starting request: urn:request:99e02191-6d24-48ce-bc20-43c1c6bedc23 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:45,702 [298] [DEBUG] [app] Ending request: urn:request:99e02191-6d24-48ce-bc20-43c1c6bedc23 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:99e02191-6d24-48ce-bc20-43c1c6bedc23', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:45,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:22:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:45,706 [285] [DEBUG] [app] Starting request: urn:request:ca5ae754-505b-4a27-b680-bd40683656f1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:45,706 [285] [DEBUG] [app] Ending request: urn:request:ca5ae754-505b-4a27-b680-bd40683656f1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ca5ae754-505b-4a27-b680-bd40683656f1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:22:45,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:45,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:45,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:45,730 [288] [DEBUG] [app] Ending request: urn:request:53b504a6-4233-40d4-b369-5f469aef2446 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:53b504a6-4233-40d4-b369-5f469aef2446', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:45,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:22:45,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:22:45,783 [286] [DEBUG] [app] Starting request: urn:request:dc0e60f4-fe51-4328-8f25-086d20ccadec (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:22:45,784 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:45,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:22:45,788 [292] [DEBUG] [app] Starting request: urn:request:096dfec7-b970-4014-ab1c-1bcc2b561c52 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:22:45,788 [292] [DEBUG] [app] Ending request: urn:request:096dfec7-b970-4014-ab1c-1bcc2b561c52 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:096dfec7-b970-4014-ab1c-1bcc2b561c52', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:22:45,789 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:45,790 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:45,792 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:22:45,792 [285] [DEBUG] [app] Starting request: urn:request:17596629-57e4-476e-a3f6-da449598e597 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:22:45,793 [285] [DEBUG] [app] Ending request: urn:request:17596629-57e4-476e-a3f6-da449598e597 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:17596629-57e4-476e-a3f6-da449598e597', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:22:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:22:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:45,793 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:45,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:45,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:45,794 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:45,801 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:45,801 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:45,811 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:45,814 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:45,817 [286] [DEBUG] [app] Ending request: urn:request:dc0e60f4-fe51-4328-8f25-086d20ccadec (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:dc0e60f4-fe51-4328-8f25-086d20ccadec', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:45,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:45,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:22:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:22:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) builder stdout | 2025-11-04 09:22:46,161 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:22:46,174 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:22:46,174 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:22:46,184 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:57.122849+00:00 (in 9.999584 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:57 GMT)" (scheduled at 2025-11-04 09:22:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 47, 123569), True, datetime.datetime(2025, 11, 4, 9, 22, 47, 123569), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:22:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:15.479769+00:00 (in 28.000260 seconds) securityworker stdout | 2025-11-04 09:22:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:17 GMT)" (scheduled at 2025-11-04 09:22:47.479080+00:00) securityworker stdout | 2025-11-04 09:22:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:47,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:47,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:22:47,496 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:47,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:47,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:47,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:47,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:47,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:47,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:47,505 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 47, 484812), 1, 2]) securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:22:47,508 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:47,509 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 47, 484812), 1, 2]) securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:47,512 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:22:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:47,512 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:22:48,001 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:22:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:22:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:33.622985+00:00 (in 44.997367 seconds) exportactionlogsworker stdout | 2025-11-04 09:22:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:48 GMT)" (scheduled at 2025-11-04 09:22:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:22:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:22:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:22:48,660 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:22:49,970 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:22:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:22:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:21.226294+00:00 (in 29.999514 seconds) autopruneworker stdout | 2025-11-04 09:22:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:21 GMT)" (scheduled at 2025-11-04 09:22:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:22:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248111235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:22:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:22:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:22:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:01.530064+00:00 (in 9.999632 seconds) notificationworker stdout | 2025-11-04 09:22:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:01 GMT)" (scheduled at 2025-11-04 09:22:51.530064+00:00) notificationworker stdout | 2025-11-04 09:22:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 51, 530749), True, datetime.datetime(2025, 11, 4, 9, 22, 51, 530749), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:22:52,252 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:55,102 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:56,184 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:22:56,198 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:22:56,198 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:22:56,208 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:07.122849+00:00 (in 9.999619 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:07 GMT)" (scheduled at 2025-11-04 09:22:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 57, 123511), True, datetime.datetime(2025, 11, 4, 9, 22, 57, 123511), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:22:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:09.133645+00:00 (in 12.001450 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:57 GMT)" (scheduled at 2025-11-04 09:22:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:22:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:22:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:22:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:22:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:22:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:27.222846+00:00 (in 29.999606 seconds) buildlogsarchiver stdout | 2025-11-04 09:22:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:27 GMT)" (scheduled at 2025-11-04 09:22:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:22:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 22, 57, 223458), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:22:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:22:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:22:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:22:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:22:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:22:57,335 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:22:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:22:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:43.432867+00:00 (in 45.908535 seconds) repositorygcworker stdout | 2025-11-04 09:22:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:57 GMT)" (scheduled at 2025-11-04 09:22:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:22:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:22:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:22:58,042 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:22:58,698 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:22:59,078 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:22:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:22:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:14.351216+00:00 (in 14.999345 seconds) quotatotalworker stdout | 2025-11-04 09:22:59,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:14 GMT)" (scheduled at 2025-11-04 09:22:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:22:59,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:22:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:22:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:22:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:22:59,454 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:59,672 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:59,851 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:59,860 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:59,952 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:00,697 [285] [DEBUG] [app] Starting request: urn:request:31cd9d29-b980-461a-af29-616aa37b8171 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:00,702 [299] [DEBUG] [app] Starting request: urn:request:f74bad45-6eeb-46c7-b951-953dab225468 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:00,702 [299] [DEBUG] [app] Ending request: urn:request:f74bad45-6eeb-46c7-b951-953dab225468 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f74bad45-6eeb-46c7-b951-953dab225468', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:23:00,703 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:00,704 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:00,706 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:00,707 [287] [DEBUG] [app] Starting request: urn:request:2a83906c-0027-42b5-a719-e01cb5c28d32 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:00,707 [287] [DEBUG] [app] Ending request: urn:request:2a83906c-0027-42b5-a719-e01cb5c28d32 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2a83906c-0027-42b5-a719-e01cb5c28d32', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:00,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:00,708 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:00,708 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:00,708 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:00,716 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:00,716 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:00,727 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:00,730 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:00,733 [285] [DEBUG] [app] Ending request: urn:request:31cd9d29-b980-461a-af29-616aa37b8171 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:31cd9d29-b980-461a-af29-616aa37b8171', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:00,733 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:00,733 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.037 47 0.037) gunicorn-web stdout | 2025-11-04 09:23:00,784 [285] [DEBUG] [app] Starting request: urn:request:8a260d98-06b6-44b8-a4d9-05738b09d81a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:00,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:00,785 [297] [DEBUG] [app] Starting request: urn:request:dc8d57b3-6b2e-49ba-858c-70c723c8873b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:00,786 [297] [DEBUG] [app] Ending request: urn:request:dc8d57b3-6b2e-49ba-858c-70c723c8873b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:dc8d57b3-6b2e-49ba-858c-70c723c8873b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:00,786 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:23:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:00,790 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:00,790 [288] [DEBUG] [app] Starting request: urn:request:d94a78d2-81c0-4589-871f-95a105d12154 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:00,791 [288] [DEBUG] [app] Ending request: urn:request:d94a78d2-81c0-4589-871f-95a105d12154 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d94a78d2-81c0-4589-871f-95a105d12154', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:00,791 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:00,791 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:00,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:00,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:00,792 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:00,799 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:00,799 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:00,809 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:00,812 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:00,815 [285] [DEBUG] [app] Ending request: urn:request:8a260d98-06b6-44b8-a4d9-05738b09d81a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8a260d98-06b6-44b8-a4d9-05738b09d81a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:00,815 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:00,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.033) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:01,337 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:11.530064+00:00 (in 9.999596 seconds) notificationworker stdout | 2025-11-04 09:23:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:11 GMT)" (scheduled at 2025-11-04 09:23:01.530064+00:00) notificationworker stdout | 2025-11-04 09:23:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 1, 530738), True, datetime.datetime(2025, 11, 4, 9, 23, 1, 530738), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:23:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:23:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:39.924652+00:00 (in 38.001413 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:23:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:01 GMT)" (scheduled at 2025-11-04 09:23:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:23:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:23:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 1, 923532), True, datetime.datetime(2025, 11, 4, 9, 23, 1, 923532), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:23:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:23:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:23:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:23:02,133 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:23:02,322 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:23:02,323 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:23:03,610 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:23:03,612 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:23:03,716 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:04,495 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:23:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:23:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:08.437104+00:00 (in 3.000887 seconds) gcworker stdout | 2025-11-04 09:23:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:35 GMT)" (scheduled at 2025-11-04 09:23:05.435809+00:00) gcworker stdout | 2025-11-04 09:23:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038585436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:23:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:23:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:23:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:23:05,804 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:23:06,179 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:06,208 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:23:06,221 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:23:06,221 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:23:06,232 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:23:06,791 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:17.122849+00:00 (in 9.999622 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:17 GMT)" (scheduled at 2025-11-04 09:23:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 7, 123525), True, datetime.datetime(2025, 11, 4, 9, 23, 7, 123525), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:07,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:23:07,198 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,153 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,154 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,168 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,199 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,213 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,226 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,249 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,396 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:08,407 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:23:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:23:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:21.434460+00:00 (in 12.998650 seconds) namespacegcworker stdout | 2025-11-04 09:23:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:28:08 GMT)" (scheduled at 2025-11-04 09:23:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:23:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 23, 8, 436057), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:23:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:23:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:35.435809+00:00 (in 26.998354 seconds) gcworker stdout | 2025-11-04 09:23:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:38 GMT)" (scheduled at 2025-11-04 09:23:08.437104+00:00) gcworker stdout | 2025-11-04 09:23:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) namespacegcworker stdout | 2025-11-04 09:23:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 23, 8, 436057), True, datetime.datetime(2025, 11, 4, 9, 23, 8, 436057), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:23:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248128449, None, 1, 0]) namespacegcworker stdout | 2025-11-04 09:23:08,452 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 23, 8, 436057), True, datetime.datetime(2025, 11, 4, 9, 23, 8, 436057), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 23, 8, 436057), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:23:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:23:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:23:08,455 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:23:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:28:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:23:08,594 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:08,596 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:08,604 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:23:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:23:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:57.131795+00:00 (in 47.997697 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:09 GMT)" (scheduled at 2025-11-04 09:23:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:23:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:23:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:23:09,349 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:23:11,117 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:15.535095+00:00 (in 4.004625 seconds) notificationworker stdout | 2025-11-04 09:23:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:21 GMT)" (scheduled at 2025-11-04 09:23:11.530064+00:00) notificationworker stdout | 2025-11-04 09:23:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 11, 530757), True, datetime.datetime(2025, 11, 4, 9, 23, 11, 530757), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:23:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:23:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:12.222828+00:00 (in 59.999559 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:23:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:12 GMT)" (scheduled at 2025-11-04 09:23:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:23:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:23:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:23:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:23:12,566 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:23:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:23:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:29.351216+00:00 (in 14.999581 seconds) quotatotalworker stdout | 2025-11-04 09:23:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:29 GMT)" (scheduled at 2025-11-04 09:23:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:23:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:23:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:23:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:23:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:23:15,064 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:23:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:23:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:17.479080+00:00 (in 1.998899 seconds) securityworker stdout | 2025-11-04 09:23:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:45 GMT)" (scheduled at 2025-11-04 09:23:15.479769+00:00) securityworker stdout | 2025-11-04 09:23:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:23:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:23:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:23:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 18, 15, 484041), 1, 2]) securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 18, 15, 484041), 1, 2]) securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:23:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:21.530064+00:00 (in 5.994535 seconds) notificationworker stdout | 2025-11-04 09:23:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:15 GMT)" (scheduled at 2025-11-04 09:23:15.535095+00:00) notificationworker stdout | 2025-11-04 09:23:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:23:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:23:15,697 [287] [DEBUG] [app] Starting request: urn:request:77deeaac-6a38-4cf4-b34e-f7578a3d293d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:15,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:15,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:15,701 [299] [DEBUG] [app] Starting request: urn:request:3c1580e8-59bf-432f-aba9-eba5ee3d7d55 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:15,701 [299] [DEBUG] [app] Ending request: urn:request:3c1580e8-59bf-432f-aba9-eba5ee3d7d55 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3c1580e8-59bf-432f-aba9-eba5ee3d7d55', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:23:15,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:15,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:15,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:15,706 [285] [DEBUG] [app] Starting request: urn:request:cd9a16b6-cfcf-449e-9bce-188bc4ea7c66 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:15,706 [285] [DEBUG] [app] Ending request: urn:request:cd9a16b6-cfcf-449e-9bce-188bc4ea7c66 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cd9a16b6-cfcf-449e-9bce-188bc4ea7c66', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:15,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:15,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:15,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:15,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:15,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:15,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:15,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:15,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:15,730 [287] [DEBUG] [app] Ending request: urn:request:77deeaac-6a38-4cf4-b34e-f7578a3d293d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:77deeaac-6a38-4cf4-b34e-f7578a3d293d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:15,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:15,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:23:15,784 [285] [DEBUG] [app] Starting request: urn:request:26f6a1a7-c43a-4698-a8cb-69f7c759b1d4 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:15,788 [297] [DEBUG] [app] Starting request: urn:request:813c448d-34e5-4243-bbfe-dbff7eb27075 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:15,788 [297] [DEBUG] [app] Ending request: urn:request:813c448d-34e5-4243-bbfe-dbff7eb27075 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:813c448d-34e5-4243-bbfe-dbff7eb27075', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:15,789 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:23:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:15,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:15,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:15,793 [287] [DEBUG] [app] Starting request: urn:request:02a605b0-ac76-4f71-9d65-dd3691065011 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:15,793 [287] [DEBUG] [app] Ending request: urn:request:02a605b0-ac76-4f71-9d65-dd3691065011 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:02a605b0-ac76-4f71-9d65-dd3691065011', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:15,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:15,794 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:15,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:15,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:15,818 [285] [DEBUG] [app] Ending request: urn:request:26f6a1a7-c43a-4698-a8cb-69f7c759b1d4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:26f6a1a7-c43a-4698-a8cb-69f7c759b1d4', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:15,818 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:15,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) builder stdout | 2025-11-04 09:23:16,232 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:23:16,246 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:23:16,246 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:23:16,256 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:27.122849+00:00 (in 9.999628 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:27 GMT)" (scheduled at 2025-11-04 09:23:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 17, 123525), True, datetime.datetime(2025, 11, 4, 9, 23, 17, 123525), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:23:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:23:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:45.479769+00:00 (in 28.000253 seconds) securityworker stdout | 2025-11-04 09:23:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:47 GMT)" (scheduled at 2025-11-04 09:23:17.479080+00:00) securityworker stdout | 2025-11-04 09:23:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:23:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:23:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:23:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:23:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:23:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:23:18,010 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:18,669 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:23:19,978 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:23:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:23:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:51.226294+00:00 (in 29.999526 seconds) autopruneworker stdout | 2025-11-04 09:23:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:51 GMT)" (scheduled at 2025-11-04 09:23:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:23:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248141234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:23:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:23:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:23:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:23:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:23:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:32.435813+00:00 (in 11.000950 seconds) namespacegcworker stdout | 2025-11-04 09:23:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:21 GMT)" (scheduled at 2025-11-04 09:23:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:23:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:23:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 21, 435166), True, datetime.datetime(2025, 11, 4, 9, 23, 21, 435166), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:23:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:23:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:23:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:23:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:31.530064+00:00 (in 9.999620 seconds) notificationworker stdout | 2025-11-04 09:23:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:31 GMT)" (scheduled at 2025-11-04 09:23:21.530064+00:00) notificationworker stdout | 2025-11-04 09:23:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 21, 530725), True, datetime.datetime(2025, 11, 4, 9, 23, 21, 530725), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:21,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:23:22,260 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:25,111 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:26,257 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:23:26,270 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:23:26,270 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:23:26,280 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:37.122849+00:00 (in 9.999628 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:37 GMT)" (scheduled at 2025-11-04 09:23:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 27, 123477), True, datetime.datetime(2025, 11, 4, 9, 23, 27, 123477), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:23:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:23:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:57.222846+00:00 (in 29.999610 seconds) buildlogsarchiver stdout | 2025-11-04 09:23:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:57 GMT)" (scheduled at 2025-11-04 09:23:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:23:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 23, 27, 223438), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:23:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:23:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:23:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:23:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:23:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:23:27,343 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:23:28,051 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:23:28,707 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:23:29,087 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:23:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:23:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:44.351216+00:00 (in 14.999591 seconds) quotatotalworker stdout | 2025-11-04 09:23:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:44 GMT)" (scheduled at 2025-11-04 09:23:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:23:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:23:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:23:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:23:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:23:29,465 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:29,690 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:29,864 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:29,874 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:29,966 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:30,696 [285] [DEBUG] [app] Starting request: urn:request:576ab404-8f1a-4068-8e40-f77454d18bd9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:30,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:30,699 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:30,700 [297] [DEBUG] [app] Starting request: urn:request:2b141355-2eb6-43ea-a833-e67f2516e515 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:30,701 [297] [DEBUG] [app] Ending request: urn:request:2b141355-2eb6-43ea-a833-e67f2516e515 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2b141355-2eb6-43ea-a833-e67f2516e515', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:30,701 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:23:30,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:30,705 [286] [DEBUG] [app] Starting request: urn:request:1fb56177-e430-4447-b79a-adae93cc35b7 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:30,705 [286] [DEBUG] [app] Ending request: urn:request:1fb56177-e430-4447-b79a-adae93cc35b7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1fb56177-e430-4447-b79a-adae93cc35b7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:30,705 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:23:30,705 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:30,713 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:30,713 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:30,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:30,726 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:30,729 [285] [DEBUG] [app] Ending request: urn:request:576ab404-8f1a-4068-8e40-f77454d18bd9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:576ab404-8f1a-4068-8e40-f77454d18bd9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:30,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:23:30,783 [286] [DEBUG] [app] Starting request: urn:request:c93e621f-6156-4318-b427-fc4e47f54a90 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:30,784 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:30,786 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:30,787 [292] [DEBUG] [app] Starting request: urn:request:73bc0ecf-93c4-4727-9b8d-b1a19bde446e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:30,788 [292] [DEBUG] [app] Ending request: urn:request:73bc0ecf-93c4-4727-9b8d-b1a19bde446e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:73bc0ecf-93c4-4727-9b8d-b1a19bde446e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:23:30,788 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:30,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:30,791 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:30,792 [288] [DEBUG] [app] Starting request: urn:request:49ebfb50-61a9-45f4-9923-67c6357e6093 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:30,792 [288] [DEBUG] [app] Ending request: urn:request:49ebfb50-61a9-45f4-9923-67c6357e6093 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:49ebfb50-61a9-45f4-9923-67c6357e6093', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:30,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:30,793 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:30,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:30,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:30,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:30,801 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:30,801 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:30,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:30,814 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:30,816 [286] [DEBUG] [app] Ending request: urn:request:c93e621f-6156-4318-b427-fc4e47f54a90 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c93e621f-6156-4318-b427-fc4e47f54a90', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:30,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:30,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:31,346 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:41.530064+00:00 (in 9.999527 seconds) notificationworker stdout | 2025-11-04 09:23:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:41 GMT)" (scheduled at 2025-11-04 09:23:31.530064+00:00) notificationworker stdout | 2025-11-04 09:23:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 31, 530817), True, datetime.datetime(2025, 11, 4, 9, 23, 31, 530817), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:23:32,142 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:23:32,331 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:23:32,330 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:23:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:23:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:21.434460+00:00 (in 48.998272 seconds) namespacegcworker stdout | 2025-11-04 09:23:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:32 GMT)" (scheduled at 2025-11-04 09:23:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:23:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:23:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:23:33,618 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:23:33,620 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:23:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:23:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:48.625192+00:00 (in 15.001643 seconds) exportactionlogsworker stdout | 2025-11-04 09:23:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:33 GMT)" (scheduled at 2025-11-04 09:23:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:23:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:23:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 33, 623844), True, datetime.datetime(2025, 11, 4, 9, 23, 33, 623844), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:23:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:23:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:23:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:33 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:23:33,725 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:34,504 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:23:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:23:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:38.437104+00:00 (in 3.000883 seconds) gcworker stdout | 2025-11-04 09:23:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:05 GMT)" (scheduled at 2025-11-04 09:23:35.435809+00:00) gcworker stdout | 2025-11-04 09:23:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038615436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:23:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:23:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:23:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:23:35,813 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:23:36,188 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:36,280 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:23:36,295 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:23:36,295 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:23:36,305 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:23:36,801 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:40.124445+00:00 (in 3.001181 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:47 GMT)" (scheduled at 2025-11-04 09:23:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 37, 123553), True, datetime.datetime(2025, 11, 4, 9, 23, 37, 123553), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:37,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:37,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:23:37,206 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,165 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,165 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,195 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,212 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,236 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,237 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,263 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,409 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:38,420 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:23:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:23:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:05.435809+00:00 (in 26.998343 seconds) gcworker stdout | 2025-11-04 09:23:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:08 GMT)" (scheduled at 2025-11-04 09:23:38.437104+00:00) gcworker stdout | 2025-11-04 09:23:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:23:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248158449, None, 1, 0]) gcworker stdout | 2025-11-04 09:23:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:23:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:23:38,605 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:38,606 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:38,614 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:23:39,358 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:23:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:23:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:01.922827+00:00 (in 21.997793 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:23:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:39 GMT)" (scheduled at 2025-11-04 09:23:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:23:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:23:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:23:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:47.122849+00:00 (in 6.997980 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:40 GMT)" (scheduled at 2025-11-04 09:23:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:23:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:23:41,126 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:51.530064+00:00 (in 9.999554 seconds) notificationworker stdout | 2025-11-04 09:23:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:51 GMT)" (scheduled at 2025-11-04 09:23:41.530064+00:00) notificationworker stdout | 2025-11-04 09:23:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 41, 530799), True, datetime.datetime(2025, 11, 4, 9, 23, 41, 530799), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:23:42,575 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:23:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:23:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:57.523915+00:00 (in 14.090611 seconds) repositorygcworker stdout | 2025-11-04 09:23:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:43 GMT)" (scheduled at 2025-11-04 09:23:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:23:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:23:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 43, 433597), True, datetime.datetime(2025, 11, 4, 9, 23, 43, 433597), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:23:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:23:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:23:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:23:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:23:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:59.351216+00:00 (in 14.999599 seconds) quotatotalworker stdout | 2025-11-04 09:23:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:59 GMT)" (scheduled at 2025-11-04 09:23:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:23:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:23:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:23:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:23:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:23:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:23:45,072 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:23:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:23:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:47.479080+00:00 (in 1.998894 seconds) securityworker stdout | 2025-11-04 09:23:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:15 GMT)" (scheduled at 2025-11-04 09:23:45.479769+00:00) securityworker stdout | 2025-11-04 09:23:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:23:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:23:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:23:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 18, 45, 483760), 1, 2]) securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 18, 45, 483760), 1, 2]) securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:23:45,696 [287] [DEBUG] [app] Starting request: urn:request:878d6c88-f46c-4f08-b1ba-f4b90b72820e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:45,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:45,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:45,701 [299] [DEBUG] [app] Starting request: urn:request:77caecec-b2ea-4805-83ce-f8e418064963 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:45,701 [299] [DEBUG] [app] Ending request: urn:request:77caecec-b2ea-4805-83ce-f8e418064963 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:77caecec-b2ea-4805-83ce-f8e418064963', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:45,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:23:45,701 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:45,702 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:45,704 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:45,705 [288] [DEBUG] [app] Starting request: urn:request:4cdc59e4-284a-44e0-8981-e4a996c69d91 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:45,705 [288] [DEBUG] [app] Ending request: urn:request:4cdc59e4-284a-44e0-8981-e4a996c69d91 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4cdc59e4-284a-44e0-8981-e4a996c69d91', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:45,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:45,706 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:45,706 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:45,706 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:45,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:45,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:45,723 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:45,726 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:45,729 [287] [DEBUG] [app] Ending request: urn:request:878d6c88-f46c-4f08-b1ba-f4b90b72820e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:878d6c88-f46c-4f08-b1ba-f4b90b72820e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:45,729 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:45,730 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:23:45,784 [285] [DEBUG] [app] Starting request: urn:request:ae2aa151-5ad3-4ab7-9d41-19ab78e2b808 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:23:45,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:45,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:23:45,789 [297] [DEBUG] [app] Starting request: urn:request:cb19dbde-4371-49e4-b854-8fa89754ed74 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:23:45,789 [297] [DEBUG] [app] Ending request: urn:request:cb19dbde-4371-49e4-b854-8fa89754ed74 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cb19dbde-4371-49e4-b854-8fa89754ed74', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:23:45,789 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:45,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:45,791 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:45,793 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:23:45,794 [286] [DEBUG] [app] Starting request: urn:request:70de8665-3b30-44f6-9132-c6bb5df75ce0 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:23:45,794 [286] [DEBUG] [app] Ending request: urn:request:70de8665-3b30-44f6-9132-c6bb5df75ce0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:70de8665-3b30-44f6-9132-c6bb5df75ce0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:45,794 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:23:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:23:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:23:45,794 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:45,795 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:45,795 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:45,795 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:45,802 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:45,802 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:45,812 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:45,815 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:45,818 [285] [DEBUG] [app] Ending request: urn:request:ae2aa151-5ad3-4ab7-9d41-19ab78e2b808 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ae2aa151-5ad3-4ab7-9d41-19ab78e2b808', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:45,819 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:45,819 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:23:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:23:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) builder stdout | 2025-11-04 09:23:46,305 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:23:46,319 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:23:46,319 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:23:46,330 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:57.122849+00:00 (in 9.999562 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:57 GMT)" (scheduled at 2025-11-04 09:23:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 47, 123601), True, datetime.datetime(2025, 11, 4, 9, 23, 47, 123601), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:23:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:23:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:15.479769+00:00 (in 28.000226 seconds) securityworker stdout | 2025-11-04 09:23:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:17 GMT)" (scheduled at 2025-11-04 09:23:47.479080+00:00) securityworker stdout | 2025-11-04 09:23:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:23:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:23:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:23:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:23:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 18, 47, 482832), 1, 2]) securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 18, 47, 482832), 1, 2]) securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:23:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:23:48,019 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:23:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:23:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:33.622985+00:00 (in 44.997377 seconds) exportactionlogsworker stdout | 2025-11-04 09:23:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:48 GMT)" (scheduled at 2025-11-04 09:23:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:23:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:23:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:23:48,678 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:23:49,987 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:23:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:23:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:21.226294+00:00 (in 29.999535 seconds) autopruneworker stdout | 2025-11-04 09:23:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:21 GMT)" (scheduled at 2025-11-04 09:23:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:23:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248171234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:23:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:23:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:23:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:23:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:01.530064+00:00 (in 9.999630 seconds) notificationworker stdout | 2025-11-04 09:23:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:01 GMT)" (scheduled at 2025-11-04 09:23:51.530064+00:00) notificationworker stdout | 2025-11-04 09:23:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 51, 530703), True, datetime.datetime(2025, 11, 4, 9, 23, 51, 530703), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:23:52,269 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:55,120 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:56,330 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:23:56,344 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:23:56,344 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:23:56,354 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:07.122849+00:00 (in 9.999594 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:07 GMT)" (scheduled at 2025-11-04 09:23:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 57, 123557), True, datetime.datetime(2025, 11, 4, 9, 23, 57, 123557), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:23:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:09.133645+00:00 (in 12.001479 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:57 GMT)" (scheduled at 2025-11-04 09:23:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:23:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:23:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:23:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:23:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:23:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:27.222846+00:00 (in 29.999572 seconds) buildlogsarchiver stdout | 2025-11-04 09:23:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:27 GMT)" (scheduled at 2025-11-04 09:23:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:23:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 23, 57, 223509), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:23:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:23:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:23:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:23:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:23:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:23:57,352 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:23:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:23:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:02.523412+00:00 (in 4.999051 seconds) repositorygcworker stdout | 2025-11-04 09:23:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:57 GMT)" (scheduled at 2025-11-04 09:23:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:23:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:23:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:23:58,060 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:23:58,716 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:23:59,096 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:23:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:23:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:14.351216+00:00 (in 14.999612 seconds) quotatotalworker stdout | 2025-11-04 09:23:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:14 GMT)" (scheduled at 2025-11-04 09:23:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:23:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:23:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:23:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:23:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:23:59,475 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:59,702 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:59,877 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:59,893 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:59,980 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:00,696 [285] [DEBUG] [app] Starting request: urn:request:6fb4f179-ae77-495f-b406-ae1327c931d4 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:00,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:00,701 [299] [DEBUG] [app] Starting request: urn:request:8be87609-109a-4bcf-91d8-88e01f1f1531 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:00,701 [299] [DEBUG] [app] Ending request: urn:request:8be87609-109a-4bcf-91d8-88e01f1f1531 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8be87609-109a-4bcf-91d8-88e01f1f1531', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:24:00,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:24:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:00,706 [288] [DEBUG] [app] Starting request: urn:request:751b3839-1b52-4efb-a84f-2111ea4a8cbf (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:00,706 [288] [DEBUG] [app] Ending request: urn:request:751b3839-1b52-4efb-a84f-2111ea4a8cbf (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:751b3839-1b52-4efb-a84f-2111ea4a8cbf', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:24:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:24:00,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:00,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:00,730 [285] [DEBUG] [app] Ending request: urn:request:6fb4f179-ae77-495f-b406-ae1327c931d4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6fb4f179-ae77-495f-b406-ae1327c931d4', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:00,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:24:00,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:24:00,784 [287] [DEBUG] [app] Starting request: urn:request:d77580a8-c955-4c09-9f0a-18695ab228ba (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:00,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:00,788 [299] [DEBUG] [app] Starting request: urn:request:fa37b879-1b1e-4d22-a4fa-51146fcdf826 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:00,788 [299] [DEBUG] [app] Ending request: urn:request:fa37b879-1b1e-4d22-a4fa-51146fcdf826 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fa37b879-1b1e-4d22-a4fa-51146fcdf826', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:24:00,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:24:00,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:00,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:00,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:00,792 [286] [DEBUG] [app] Starting request: urn:request:d3d541bf-db70-4d24-8a29-85af58e16271 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:00,793 [286] [DEBUG] [app] Ending request: urn:request:d3d541bf-db70-4d24-8a29-85af58e16271 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d3d541bf-db70-4d24-8a29-85af58e16271', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:24:00,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:00,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:00,794 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:00,794 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:00,802 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:00,802 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:00,812 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:00,815 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:00,818 [287] [DEBUG] [app] Ending request: urn:request:d77580a8-c955-4c09-9f0a-18695ab228ba (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d77580a8-c955-4c09-9f0a-18695ab228ba', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:00,818 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:24:00,818 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) manifestsubjectbackfillworker stdout | 2025-11-04 09:24:01,355 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:24:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:11.530064+00:00 (in 9.999581 seconds) notificationworker stdout | 2025-11-04 09:24:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:11 GMT)" (scheduled at 2025-11-04 09:24:01.530064+00:00) notificationworker stdout | 2025-11-04 09:24:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:24:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 1, 531019), True, datetime.datetime(2025, 11, 4, 9, 24, 1, 531019), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:24:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:24:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:24:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:24:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:24:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:39.924652+00:00 (in 38.001454 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:24:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:01 GMT)" (scheduled at 2025-11-04 09:24:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:24:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:24:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 1, 923425), True, datetime.datetime(2025, 11, 4, 9, 24, 1, 923425), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:24:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:24:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:24:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:24:02,151 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:24:02,340 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:24:02,350 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:24:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:24:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:43.432867+00:00 (in 40.909031 seconds) repositorygcworker stdout | 2025-11-04 09:24:02,523 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:29:02 GMT)" (scheduled at 2025-11-04 09:24:02.523412+00:00) repositorygcworker stdout | 2025-11-04 09:24:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 24, 2, 524072), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:24:02,535 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 24, 2, 524072), True, datetime.datetime(2025, 11, 4, 9, 24, 2, 524072), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:24:02,539 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 24, 2, 524072), True, datetime.datetime(2025, 11, 4, 9, 24, 2, 524072), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 24, 2, 524072), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:24:02,542 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:24:02,543 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:29:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:24:03,627 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:24:03,629 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:24:03,734 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:24:04,513 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:24:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:24:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:08.437104+00:00 (in 3.000884 seconds) gcworker stdout | 2025-11-04 09:24:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:35 GMT)" (scheduled at 2025-11-04 09:24:05.435809+00:00) gcworker stdout | 2025-11-04 09:24:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038645436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:24:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:24:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:24:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:24:05,822 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:24:06,197 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:24:06,354 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:24:06,368 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:24:06,368 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:24:06,378 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:24:06,810 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:24:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:17.122849+00:00 (in 9.999601 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:17 GMT)" (scheduled at 2025-11-04 09:24:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:24:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 7, 123521), True, datetime.datetime(2025, 11, 4, 9, 24, 7, 123521), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:24:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:24:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:24:07,215 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,179 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,188 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,209 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,223 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,248 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,248 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,275 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,431 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:08,431 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:24:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:24:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:35.435809+00:00 (in 26.998343 seconds) gcworker stdout | 2025-11-04 09:24:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:38 GMT)" (scheduled at 2025-11-04 09:24:08.437104+00:00) gcworker stdout | 2025-11-04 09:24:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:24:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248188449, None, 1, 0]) gcworker stdout | 2025-11-04 09:24:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:24:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:38 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:24:08,615 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:24:08,616 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:24:08,624 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:24:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:24:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:57.131795+00:00 (in 47.997757 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:24:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:09 GMT)" (scheduled at 2025-11-04 09:24:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:24:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:24:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:24:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:24:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:24:09,367 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:24:11,135 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:24:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:15.535095+00:00 (in 4.004625 seconds) notificationworker stdout | 2025-11-04 09:24:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:21 GMT)" (scheduled at 2025-11-04 09:24:11.530064+00:00) notificationworker stdout | 2025-11-04 09:24:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:24:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 11, 530744), True, datetime.datetime(2025, 11, 4, 9, 24, 11, 530744), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:24:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:24:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:24:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:24:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:24:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:12.222828+00:00 (in 59.999562 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:24:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:12 GMT)" (scheduled at 2025-11-04 09:24:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:24:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:24:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:24:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:24:12,584 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:24:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:24:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:29.351216+00:00 (in 14.999609 seconds) quotatotalworker stdout | 2025-11-04 09:24:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:29 GMT)" (scheduled at 2025-11-04 09:24:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:24:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:24:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:24:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:24:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:24:15,081 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:24:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:24:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:17.479080+00:00 (in 1.998877 seconds) securityworker stdout | 2025-11-04 09:24:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:45 GMT)" (scheduled at 2025-11-04 09:24:15.479769+00:00) securityworker stdout | 2025-11-04 09:24:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:24:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:24:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:24:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:24:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 19, 15, 484376), 1, 2]) securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 19, 15, 484376), 1, 2]) securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:24:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:21.530064+00:00 (in 5.994543 seconds) notificationworker stdout | 2025-11-04 09:24:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:15 GMT)" (scheduled at 2025-11-04 09:24:15.535095+00:00) notificationworker stdout | 2025-11-04 09:24:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:24:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:24:15,697 [286] [DEBUG] [app] Starting request: urn:request:1674b237-a8e0-4f7e-a30c-6267f24258d6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:15,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:15,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:15,702 [299] [DEBUG] [app] Starting request: urn:request:ee9ac05d-7501-4c3c-846a-f794d40d74cc (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:15,702 [299] [DEBUG] [app] Ending request: urn:request:ee9ac05d-7501-4c3c-846a-f794d40d74cc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ee9ac05d-7501-4c3c-846a-f794d40d74cc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:24:15,703 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:24:15,703 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:15,704 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:15,706 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:15,707 [288] [DEBUG] [app] Starting request: urn:request:631cc718-8573-4c77-b793-c54c3b99d7b1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:15,707 [288] [DEBUG] [app] Ending request: urn:request:631cc718-8573-4c77-b793-c54c3b99d7b1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:631cc718-8573-4c77-b793-c54c3b99d7b1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:24:15,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:15,708 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:15,708 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:15,708 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:15,708 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:15,716 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:15,716 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:15,726 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:15,729 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:15,732 [286] [DEBUG] [app] Ending request: urn:request:1674b237-a8e0-4f7e-a30c-6267f24258d6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1674b237-a8e0-4f7e-a30c-6267f24258d6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:15,733 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:24:15,733 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.037 47 0.036) gunicorn-web stdout | 2025-11-04 09:24:15,784 [288] [DEBUG] [app] Starting request: urn:request:182455dd-33cd-4493-92ef-df51086d5f59 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:15,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:15,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:15,788 [299] [DEBUG] [app] Starting request: urn:request:54a3f103-6e6f-40e7-a427-1fed4762ce52 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:15,788 [299] [DEBUG] [app] Ending request: urn:request:54a3f103-6e6f-40e7-a427-1fed4762ce52 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:54a3f103-6e6f-40e7-a427-1fed4762ce52', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:24:15,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:24:15,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:15,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:15,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:15,792 [285] [DEBUG] [app] Starting request: urn:request:c0715e11-c1cf-4779-b989-aa3e5afc2b66 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:15,793 [285] [DEBUG] [app] Ending request: urn:request:c0715e11-c1cf-4779-b989-aa3e5afc2b66 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c0715e11-c1cf-4779-b989-aa3e5afc2b66', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:24:15,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:15,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:15,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:15,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:15,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:15,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:15,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:15,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:15,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:15,817 [288] [DEBUG] [app] Ending request: urn:request:182455dd-33cd-4493-92ef-df51086d5f59 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:182455dd-33cd-4493-92ef-df51086d5f59', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:15,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:24:15,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) builder stdout | 2025-11-04 09:24:16,378 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:24:16,391 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:24:16,391 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:24:16,401 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:27.122849+00:00 (in 9.999645 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:27 GMT)" (scheduled at 2025-11-04 09:24:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:24:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 17, 123505), True, datetime.datetime(2025, 11, 4, 9, 24, 17, 123505), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:24:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:24:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:24:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:24:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:45.479769+00:00 (in 28.000209 seconds) securityworker stdout | 2025-11-04 09:24:17,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:47 GMT)" (scheduled at 2025-11-04 09:24:17.479080+00:00) securityworker stdout | 2025-11-04 09:24:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:24:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:24:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:24:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:24:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:24:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:24:18,028 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:24:18,687 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:24:19,995 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:24:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:24:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:51.226294+00:00 (in 29.999545 seconds) autopruneworker stdout | 2025-11-04 09:24:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:51 GMT)" (scheduled at 2025-11-04 09:24:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:24:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248201234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:24:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:24:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:24:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:24:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:24:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:32.435813+00:00 (in 11.000957 seconds) namespacegcworker stdout | 2025-11-04 09:24:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:21 GMT)" (scheduled at 2025-11-04 09:24:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:24:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:24:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 21, 435237), True, datetime.datetime(2025, 11, 4, 9, 24, 21, 435237), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:24:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:24:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:24:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:24:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:31.530064+00:00 (in 9.999602 seconds) notificationworker stdout | 2025-11-04 09:24:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:31 GMT)" (scheduled at 2025-11-04 09:24:21.530064+00:00) notificationworker stdout | 2025-11-04 09:24:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:24:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 21, 530763), True, datetime.datetime(2025, 11, 4, 9, 24, 21, 530763), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:24:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:24:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:24:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:24:22,278 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:24:25,128 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:24:26,401 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:24:26,415 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:24:26,415 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:24:26,425 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:37.122849+00:00 (in 9.999629 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:37 GMT)" (scheduled at 2025-11-04 09:24:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:24:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 27, 123510), True, datetime.datetime(2025, 11, 4, 9, 24, 27, 123510), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:24:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:24:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:24:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:24:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:57.222846+00:00 (in 29.999573 seconds) buildlogsarchiver stdout | 2025-11-04 09:24:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:57 GMT)" (scheduled at 2025-11-04 09:24:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:24:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 24, 27, 223508), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:24:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:24:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:24:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:24:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:24:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:24:27,361 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:24:28,069 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:24:28,725 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:24:29,105 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:24:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:24:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:44.351216+00:00 (in 14.999550 seconds) quotatotalworker stdout | 2025-11-04 09:24:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:44 GMT)" (scheduled at 2025-11-04 09:24:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:24:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:24:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:24:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:24:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:24:29,486 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:29,713 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:29,891 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:29,905 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:29,993 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:30,697 [286] [DEBUG] [app] Starting request: urn:request:192705b4-6e11-4e54-8789-a1397b671065 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:30,697 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:30,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:30,701 [292] [DEBUG] [app] Starting request: urn:request:07b9e10f-38da-4063-a9e9-ff69369d93a1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:30,701 [292] [DEBUG] [app] Ending request: urn:request:07b9e10f-38da-4063-a9e9-ff69369d93a1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:07b9e10f-38da-4063-a9e9-ff69369d93a1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:24:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:24:30,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:30,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:30,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:30,705 [285] [DEBUG] [app] Starting request: urn:request:05271d4c-807b-4e3e-90be-f4354c9e8ebb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:30,706 [285] [DEBUG] [app] Ending request: urn:request:05271d4c-807b-4e3e-90be-f4354c9e8ebb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:05271d4c-807b-4e3e-90be-f4354c9e8ebb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:24:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:30,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:30,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:30,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:30,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:30,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:30,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:30,723 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:30,726 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:30,729 [286] [DEBUG] [app] Ending request: urn:request:192705b4-6e11-4e54-8789-a1397b671065 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:192705b4-6e11-4e54-8789-a1397b671065', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:30,730 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:24:30,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:24:30,783 [285] [DEBUG] [app] Starting request: urn:request:74e1e19f-7854-4e48-92c6-98c9fc8f42a1 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:30,786 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:30,787 [299] [DEBUG] [app] Starting request: urn:request:2a08dd2e-88e6-44b1-bc90-54c8e0584701 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:30,787 [299] [DEBUG] [app] Ending request: urn:request:2a08dd2e-88e6-44b1-bc90-54c8e0584701 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2a08dd2e-88e6-44b1-bc90-54c8e0584701', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:24:30,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:24:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:30,792 [287] [DEBUG] [app] Starting request: urn:request:31d7c831-ffa1-40e4-9ae8-10b64b8d0444 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:30,792 [287] [DEBUG] [app] Ending request: urn:request:31d7c831-ffa1-40e4-9ae8-10b64b8d0444 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:31d7c831-ffa1-40e4-9ae8-10b64b8d0444', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:30,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:24:30,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:30,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:30,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:30,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:30,817 [285] [DEBUG] [app] Ending request: urn:request:74e1e19f-7854-4e48-92c6-98c9fc8f42a1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:74e1e19f-7854-4e48-92c6-98c9fc8f42a1', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:30,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:24:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:24:31,363 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:24:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:40.533243+00:00 (in 9.002763 seconds) notificationworker stdout | 2025-11-04 09:24:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:41 GMT)" (scheduled at 2025-11-04 09:24:31.530064+00:00) notificationworker stdout | 2025-11-04 09:24:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:24:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 31, 530757), True, datetime.datetime(2025, 11, 4, 9, 24, 31, 530757), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:24:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:24:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:24:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:24:32,160 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:24:32,349 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:24:32,359 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:24:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:24:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:21.434460+00:00 (in 48.998249 seconds) namespacegcworker stdout | 2025-11-04 09:24:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:32 GMT)" (scheduled at 2025-11-04 09:24:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:24:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:24:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:32 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:24:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:24:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:48.625192+00:00 (in 15.001838 seconds) exportactionlogsworker stdout | 2025-11-04 09:24:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:33 GMT)" (scheduled at 2025-11-04 09:24:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:24:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:24:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 33, 623594), True, datetime.datetime(2025, 11, 4, 9, 24, 33, 623594), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:24:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:24:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:24:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:24:33,636 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:24:33,638 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:24:33,743 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:24:34,522 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:24:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:24:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:38.437104+00:00 (in 3.000889 seconds) gcworker stdout | 2025-11-04 09:24:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:05 GMT)" (scheduled at 2025-11-04 09:24:35.435809+00:00) gcworker stdout | 2025-11-04 09:24:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038675436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:24:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:24:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:24:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:24:35,831 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:24:36,206 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:24:36,425 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:24:36,439 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:24:36,439 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:24:36,449 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:24:36,819 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:24:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:40.124445+00:00 (in 3.001164 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:47 GMT)" (scheduled at 2025-11-04 09:24:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:24:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 37, 123590), True, datetime.datetime(2025, 11, 4, 9, 24, 37, 123590), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:24:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:24:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:24:37,224 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,193 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,200 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,222 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,234 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,257 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,259 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,287 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:24:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:24:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:05.435809+00:00 (in 26.998341 seconds) gcworker stdout | 2025-11-04 09:24:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:08 GMT)" (scheduled at 2025-11-04 09:24:38.437104+00:00) gcworker stdout | 2025-11-04 09:24:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:24:38,443 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:24:38,446 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:24:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248218449, None, 1, 0]) gcworker stdout | 2025-11-04 09:24:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:24:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:08 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:24:38,634 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:24:38,626 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:24:38,626 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:24:39,376 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:24:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:24:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:01.922827+00:00 (in 21.997749 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:24:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:39 GMT)" (scheduled at 2025-11-04 09:24:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:24:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:24:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:24:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:47.122849+00:00 (in 6.997979 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:40 GMT)" (scheduled at 2025-11-04 09:24:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:24:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:24:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:41.530064+00:00 (in 0.996339 seconds) notificationworker stdout | 2025-11-04 09:24:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:29:40 GMT)" (scheduled at 2025-11-04 09:24:40.533243+00:00) notificationworker stdout | 2025-11-04 09:24:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 24, 40, 533995), 'notification/%']) notificationworker stdout | 2025-11-04 09:24:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 24, 40, 533995), True, datetime.datetime(2025, 11, 4, 9, 24, 40, 533995), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:24:40,550 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 24, 40, 533995), True, datetime.datetime(2025, 11, 4, 9, 24, 40, 533995), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 24, 40, 533995), 'notification/%']) notificationworker stdout | 2025-11-04 09:24:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:24:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:29:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:24:41,143 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:24:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:51.530064+00:00 (in 9.999542 seconds) notificationworker stdout | 2025-11-04 09:24:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:51 GMT)" (scheduled at 2025-11-04 09:24:41.530064+00:00) notificationworker stdout | 2025-11-04 09:24:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:24:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 41, 530809), True, datetime.datetime(2025, 11, 4, 9, 24, 41, 530809), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:24:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:24:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:24:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:24:42,593 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:24:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:24:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:57.523915+00:00 (in 14.090642 seconds) repositorygcworker stdout | 2025-11-04 09:24:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:43 GMT)" (scheduled at 2025-11-04 09:24:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:24:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:24:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 43, 433511), True, datetime.datetime(2025, 11, 4, 9, 24, 43, 433511), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:24:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:24:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:24:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:24:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:24:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:59.351216+00:00 (in 14.999618 seconds) quotatotalworker stdout | 2025-11-04 09:24:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:59 GMT)" (scheduled at 2025-11-04 09:24:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:24:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:24:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:24:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:24:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:24:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:24:45,090 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:24:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:24:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:47.479080+00:00 (in 1.998916 seconds) securityworker stdout | 2025-11-04 09:24:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:15 GMT)" (scheduled at 2025-11-04 09:24:45.479769+00:00) securityworker stdout | 2025-11-04 09:24:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:24:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:24:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:24:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:45,493 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:45,493 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:24:45,493 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:45,493 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:45,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 19, 45, 483571), 1, 2]) securityworker stdout | 2025-11-04 09:24:45,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:45,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:45,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:45,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:45,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 19, 45, 483571), 1, 2]) securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:24:45,696 [285] [DEBUG] [app] Starting request: urn:request:96e3dd36-77d0-4715-a17c-2f560fd7d6d4 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:45,701 [299] [DEBUG] [app] Starting request: urn:request:14df1291-3af9-4fc7-86e3-19dd42fe416b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:45,701 [299] [DEBUG] [app] Ending request: urn:request:14df1291-3af9-4fc7-86e3-19dd42fe416b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:14df1291-3af9-4fc7-86e3-19dd42fe416b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:24:45,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:24:45,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:45,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:45,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:45,705 [288] [DEBUG] [app] Starting request: urn:request:a0c9fb8f-ba5c-4d40-9745-7e0fef410fc6 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:45,705 [288] [DEBUG] [app] Ending request: urn:request:a0c9fb8f-ba5c-4d40-9745-7e0fef410fc6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a0c9fb8f-ba5c-4d40-9745-7e0fef410fc6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:24:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:45,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:45,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:45,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:45,713 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:45,713 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:45,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:45,726 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:45,729 [285] [DEBUG] [app] Ending request: urn:request:96e3dd36-77d0-4715-a17c-2f560fd7d6d4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:96e3dd36-77d0-4715-a17c-2f560fd7d6d4', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:45,729 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:24:45,729 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:24:45,783 [288] [DEBUG] [app] Starting request: urn:request:adb6886d-ce3c-4c24-8fcb-36d09ba7f017 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:24:45,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:45,786 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:24:45,787 [292] [DEBUG] [app] Starting request: urn:request:b4f7acd1-696c-49ff-a028-7fc13a19bd60 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:24:45,788 [292] [DEBUG] [app] Ending request: urn:request:b4f7acd1-696c-49ff-a028-7fc13a19bd60 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b4f7acd1-696c-49ff-a028-7fc13a19bd60', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:24:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:24:45,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:45,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:24:45,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:24:45,791 [285] [DEBUG] [app] Starting request: urn:request:a7658af5-0739-4b60-bf63-789c5470a2bc (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:24:45,792 [285] [DEBUG] [app] Ending request: urn:request:a7658af5-0739-4b60-bf63-789c5470a2bc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a7658af5-0739-4b60-bf63-789c5470a2bc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:24:45,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:24:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:24:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:24:45,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:24:45,792 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:24:45,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:24:45,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:24:45,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:24:45,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:24:45,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:24:45,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:24:45,816 [288] [DEBUG] [app] Ending request: urn:request:adb6886d-ce3c-4c24-8fcb-36d09ba7f017 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:adb6886d-ce3c-4c24-8fcb-36d09ba7f017', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:24:45,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:24:45,816 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:24:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:24:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) builder stdout | 2025-11-04 09:24:46,449 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:24:46,463 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:24:46,463 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:24:46,473 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:53.124163+00:00 (in 6.000887 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:57 GMT)" (scheduled at 2025-11-04 09:24:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:24:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 47, 123557), True, datetime.datetime(2025, 11, 4, 9, 24, 47, 123557), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:24:47,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:24:47,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:47,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:24:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:24:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:24:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:15.479769+00:00 (in 28.000216 seconds) securityworker stdout | 2025-11-04 09:24:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:17 GMT)" (scheduled at 2025-11-04 09:24:47.479080+00:00) securityworker stdout | 2025-11-04 09:24:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:24:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:24:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:24:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:24:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:24:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 19, 47, 483337), 1, 2]) securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:24:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:24:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 19, 47, 483337), 1, 2]) securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:24:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:24:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:24:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:24:48,036 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:24:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:24:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:33.622985+00:00 (in 44.997369 seconds) exportactionlogsworker stdout | 2025-11-04 09:24:48,626 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:48 GMT)" (scheduled at 2025-11-04 09:24:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:24:48,626 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:24:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:24:48,696 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:24:50,004 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:24:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:24:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:21.226294+00:00 (in 29.999582 seconds) autopruneworker stdout | 2025-11-04 09:24:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:21 GMT)" (scheduled at 2025-11-04 09:24:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:24:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248231234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:24:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:24:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:24:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:24:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:24:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:01.530064+00:00 (in 9.999601 seconds) notificationworker stdout | 2025-11-04 09:24:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:01 GMT)" (scheduled at 2025-11-04 09:24:51.530064+00:00) notificationworker stdout | 2025-11-04 09:24:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:24:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 51, 530734), True, datetime.datetime(2025, 11, 4, 9, 24, 51, 530734), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:24:51,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:24:51,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:24:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:24:52,287 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:24:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:57.122849+00:00 (in 3.998236 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:29:53 GMT)" (scheduled at 2025-11-04 09:24:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 24, 53, 124894), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:24:53,137 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 24, 53, 124894), True, datetime.datetime(2025, 11, 4, 9, 24, 53, 124894), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:24:53,141 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 24, 53, 124894), True, datetime.datetime(2025, 11, 4, 9, 24, 53, 124894), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 24, 53, 124894), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:24:53,144 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:53,144 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:29:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:24:55,137 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:24:56,473 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:24:56,487 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:24:56,487 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:24:56,497 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:24:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:07.122849+00:00 (in 9.999574 seconds) proxycacheblobworker stdout | 2025-11-04 09:24:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:07 GMT)" (scheduled at 2025-11-04 09:24:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:24:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:24:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 24, 57, 123596), True, datetime.datetime(2025, 11, 4, 9, 24, 57, 123596), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:24:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:24:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:09.133645+00:00 (in 12.001467 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:24:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:57 GMT)" (scheduled at 2025-11-04 09:24:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:24:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:24:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:24:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:24:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:24:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:24:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:24:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:24:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:24:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:27.222846+00:00 (in 29.999584 seconds) buildlogsarchiver stdout | 2025-11-04 09:24:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:27 GMT)" (scheduled at 2025-11-04 09:24:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:24:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 24, 57, 223450), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:24:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:24:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:24:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:24:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:24:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:24:57,370 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:24:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:24:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:43.432867+00:00 (in 45.908522 seconds) repositorygcworker stdout | 2025-11-04 09:24:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:57 GMT)" (scheduled at 2025-11-04 09:24:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:24:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:24:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:25:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:24:58,089 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:24:58,745 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:24:59,126 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:24:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:24:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:14.351216+00:00 (in 14.999573 seconds) quotatotalworker stdout | 2025-11-04 09:24:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:14 GMT)" (scheduled at 2025-11-04 09:24:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:24:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:24:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:24:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:24:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:24:59,496 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:59,725 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:59,904 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:24:59,918 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:00,006 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:00,697 [287] [DEBUG] [app] Starting request: urn:request:ed06b9a9-f39c-40f9-8278-c98d52389746 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:00,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:00,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:00,701 [296] [DEBUG] [app] Starting request: urn:request:cd5e72bd-e1c8-4793-9c3f-19066913044c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:00,702 [296] [DEBUG] [app] Ending request: urn:request:cd5e72bd-e1c8-4793-9c3f-19066913044c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cd5e72bd-e1c8-4793-9c3f-19066913044c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:25:00,702 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:25:00,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:00,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:00,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:00,706 [288] [DEBUG] [app] Starting request: urn:request:2ecc4b71-9a50-49a0-82b3-d0229aa3d09b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:00,706 [288] [DEBUG] [app] Ending request: urn:request:2ecc4b71-9a50-49a0-82b3-d0229aa3d09b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2ecc4b71-9a50-49a0-82b3-d0229aa3d09b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:25:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:25:00,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:00,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:00,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:00,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:00,715 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:00,715 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:00,725 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:00,728 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:00,731 [287] [DEBUG] [app] Ending request: urn:request:ed06b9a9-f39c-40f9-8278-c98d52389746 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ed06b9a9-f39c-40f9-8278-c98d52389746', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:00,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:25:00,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:25:00,783 [288] [DEBUG] [app] Starting request: urn:request:7cabbec4-e421-49af-8bd3-377cb4e09908 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:00,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:00,786 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:00,787 [299] [DEBUG] [app] Starting request: urn:request:7b131445-8279-4aba-85d8-5ac178e14d24 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:00,788 [299] [DEBUG] [app] Ending request: urn:request:7b131445-8279-4aba-85d8-5ac178e14d24 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7b131445-8279-4aba-85d8-5ac178e14d24', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:25:00,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:25:00,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:00,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:00,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:00,791 [287] [DEBUG] [app] Starting request: urn:request:c7b6f0ce-f16e-4dc4-a0a3-1026a6f21b08 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:00,792 [287] [DEBUG] [app] Ending request: urn:request:c7b6f0ce-f16e-4dc4-a0a3-1026a6f21b08 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c7b6f0ce-f16e-4dc4-a0a3-1026a6f21b08', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:25:00,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:25:00,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:00,792 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:00,792 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:00,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:00,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:00,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:00,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:00,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:00,816 [288] [DEBUG] [app] Ending request: urn:request:7cabbec4-e421-49af-8bd3-377cb4e09908 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7cabbec4-e421-49af-8bd3-377cb4e09908', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:00,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:25:00,816 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:01,372 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:25:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:25:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:11.530064+00:00 (in 9.999600 seconds) notificationworker stdout | 2025-11-04 09:25:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:11 GMT)" (scheduled at 2025-11-04 09:25:01.530064+00:00) notificationworker stdout | 2025-11-04 09:25:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:25:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 1, 531006), True, datetime.datetime(2025, 11, 4, 9, 25, 1, 531006), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:25:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:25:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:25:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:25:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:25:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:39.924652+00:00 (in 38.001431 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:25:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:01 GMT)" (scheduled at 2025-11-04 09:25:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:25:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:25:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 1, 923477), True, datetime.datetime(2025, 11, 4, 9, 25, 1, 923477), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:25:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:25:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:25:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:25:02,169 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:25:02,357 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:25:02,368 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:25:03,645 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:25:03,659 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:25:03,751 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:25:04,531 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:25:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:25:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:08.437104+00:00 (in 3.000868 seconds) gcworker stdout | 2025-11-04 09:25:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:35 GMT)" (scheduled at 2025-11-04 09:25:05.435809+00:00) gcworker stdout | 2025-11-04 09:25:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038705436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:25:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:25:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:25:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:25:05,840 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:25:06,215 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:25:06,497 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:25:06,511 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:25:06,511 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:25:06,528 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:25:06,828 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:25:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:25:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:17.122849+00:00 (in 9.999553 seconds) proxycacheblobworker stdout | 2025-11-04 09:25:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:17 GMT)" (scheduled at 2025-11-04 09:25:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:25:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:25:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 7, 123581), True, datetime.datetime(2025, 11, 4, 9, 25, 7, 123581), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:25:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:25:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:25:07,234 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,210 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,217 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,235 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,244 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,269 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,272 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,298 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:25:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:25:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:35.435809+00:00 (in 26.998207 seconds) gcworker stdout | 2025-11-04 09:25:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:38 GMT)" (scheduled at 2025-11-04 09:25:08.437104+00:00) gcworker stdout | 2025-11-04 09:25:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:25:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248248449, None, 1, 0]) gcworker stdout | 2025-11-04 09:25:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:25:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:25:08,456 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:08,460 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:25:08,645 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:25:08,651 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:25:08,652 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:25:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:25:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:57.131795+00:00 (in 47.997721 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:09 GMT)" (scheduled at 2025-11-04 09:25:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:25:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:25:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:25:09,384 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:25:11,152 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:25:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:25:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:15.535095+00:00 (in 4.004589 seconds) notificationworker stdout | 2025-11-04 09:25:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:21 GMT)" (scheduled at 2025-11-04 09:25:11.530064+00:00) notificationworker stdout | 2025-11-04 09:25:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:25:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 11, 530805), True, datetime.datetime(2025, 11, 4, 9, 25, 11, 530805), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:25:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:25:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:25:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:25:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:25:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:12.222828+00:00 (in 59.999519 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:25:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:12 GMT)" (scheduled at 2025-11-04 09:25:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:25:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:25:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:25:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:25:12,601 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:25:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:25:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:29.351216+00:00 (in 14.999595 seconds) quotatotalworker stdout | 2025-11-04 09:25:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:29 GMT)" (scheduled at 2025-11-04 09:25:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:25:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:25:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:25:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:25:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:25:15,098 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:25:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:25:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:17.479080+00:00 (in 1.998901 seconds) securityworker stdout | 2025-11-04 09:25:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:45 GMT)" (scheduled at 2025-11-04 09:25:15.479769+00:00) securityworker stdout | 2025-11-04 09:25:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:25:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:25:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:25:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 20, 15, 483993), 1, 2]) securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 20, 15, 483993), 1, 2]) securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:25:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:25:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:25:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:21.530064+00:00 (in 5.994513 seconds) notificationworker stdout | 2025-11-04 09:25:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:15 GMT)" (scheduled at 2025-11-04 09:25:15.535095+00:00) notificationworker stdout | 2025-11-04 09:25:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:25:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:25:15,697 [285] [DEBUG] [app] Starting request: urn:request:7dcb82ff-7b6f-4ebc-bdb5-93e6add5f6d8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:15,701 [298] [DEBUG] [app] Starting request: urn:request:87866696-aeaa-46cc-a8f9-bc32bbed069a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:15,702 [298] [DEBUG] [app] Ending request: urn:request:87866696-aeaa-46cc-a8f9-bc32bbed069a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:87866696-aeaa-46cc-a8f9-bc32bbed069a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:25:15,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:25:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:15,706 [288] [DEBUG] [app] Starting request: urn:request:f50117ac-9983-48be-8316-06807bd73eae (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:15,706 [288] [DEBUG] [app] Ending request: urn:request:f50117ac-9983-48be-8316-06807bd73eae (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f50117ac-9983-48be-8316-06807bd73eae', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:25:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:25:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:25:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:15,730 [285] [DEBUG] [app] Ending request: urn:request:7dcb82ff-7b6f-4ebc-bdb5-93e6add5f6d8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7dcb82ff-7b6f-4ebc-bdb5-93e6add5f6d8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:15,730 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:25:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:25:15,784 [285] [DEBUG] [app] Starting request: urn:request:3b6d8122-4408-4333-a4c5-8a9e15900269 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:15,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:15,785 [298] [DEBUG] [app] Starting request: urn:request:cb87d376-ae38-4614-a6bc-54626e4051f1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:15,786 [298] [DEBUG] [app] Ending request: urn:request:cb87d376-ae38-4614-a6bc-54626e4051f1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cb87d376-ae38-4614-a6bc-54626e4051f1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:25:15,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:25:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:15,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:15,790 [288] [DEBUG] [app] Starting request: urn:request:b023c637-7227-4e2c-ac33-da1c6b6cc203 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:15,790 [288] [DEBUG] [app] Ending request: urn:request:b023c637-7227-4e2c-ac33-da1c6b6cc203 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b023c637-7227-4e2c-ac33-da1c6b6cc203', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:25:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:25:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:25:15,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:15,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:15,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:15,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:15,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:15,814 [285] [DEBUG] [app] Ending request: urn:request:3b6d8122-4408-4333-a4c5-8a9e15900269 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3b6d8122-4408-4333-a4c5-8a9e15900269', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:15,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:25:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) builder stdout | 2025-11-04 09:25:16,528 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:25:16,545 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:25:16,545 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:25:16,555 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:25:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:27.122849+00:00 (in 9.999613 seconds) proxycacheblobworker stdout | 2025-11-04 09:25:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:27 GMT)" (scheduled at 2025-11-04 09:25:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:25:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:25:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 17, 123466), True, datetime.datetime(2025, 11, 4, 9, 25, 17, 123466), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:25:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:25:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:25:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:25:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:45.479769+00:00 (in 28.000220 seconds) securityworker stdout | 2025-11-04 09:25:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:47 GMT)" (scheduled at 2025-11-04 09:25:17.479080+00:00) securityworker stdout | 2025-11-04 09:25:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:25:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:25:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:25:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:25:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:25:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:25:18,045 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:25:18,705 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:25:20,013 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:25:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:25:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:51.226294+00:00 (in 29.999606 seconds) autopruneworker stdout | 2025-11-04 09:25:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:51 GMT)" (scheduled at 2025-11-04 09:25:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:25:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248261234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:25:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:25:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:25:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:25:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:25:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:32.435813+00:00 (in 11.000980 seconds) namespacegcworker stdout | 2025-11-04 09:25:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:21 GMT)" (scheduled at 2025-11-04 09:25:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:25:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:25:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 21, 435111), True, datetime.datetime(2025, 11, 4, 9, 25, 21, 435111), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:25:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:25:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:25:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:25:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:25:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:31.530064+00:00 (in 9.999609 seconds) notificationworker stdout | 2025-11-04 09:25:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:31 GMT)" (scheduled at 2025-11-04 09:25:21.530064+00:00) notificationworker stdout | 2025-11-04 09:25:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:25:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 21, 530712), True, datetime.datetime(2025, 11, 4, 9, 25, 21, 530712), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:25:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:25:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:25:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:25:22,296 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:25:25,146 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:25:26,555 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:25:26,569 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:25:26,569 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:25:26,579 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:25:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:37.122849+00:00 (in 9.999620 seconds) proxycacheblobworker stdout | 2025-11-04 09:25:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:37 GMT)" (scheduled at 2025-11-04 09:25:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:25:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:25:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 27, 123463), True, datetime.datetime(2025, 11, 4, 9, 25, 27, 123463), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:25:27,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:25:27,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:27,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:25:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:25:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:57.222846+00:00 (in 29.999571 seconds) buildlogsarchiver stdout | 2025-11-04 09:25:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:57 GMT)" (scheduled at 2025-11-04 09:25:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:25:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 25, 27, 223533), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:25:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:25:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:25:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:25:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:25:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:25:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:25:27,379 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:25:28,098 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:25:28,753 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:25:29,135 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:25:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:25:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:44.351216+00:00 (in 14.999548 seconds) quotatotalworker stdout | 2025-11-04 09:25:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:44 GMT)" (scheduled at 2025-11-04 09:25:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:25:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:25:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:25:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:25:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:25:29,507 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:29,737 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:29,916 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:29,931 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:30,019 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:30,697 [288] [DEBUG] [app] Starting request: urn:request:1a23d6b3-1f87-4af6-9759-365ae50ea394 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:30,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:30,702 [292] [DEBUG] [app] Starting request: urn:request:9726ca2e-b2a0-4371-96e7-ff9bbdcdf6d0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:30,702 [292] [DEBUG] [app] Ending request: urn:request:9726ca2e-b2a0-4371-96e7-ff9bbdcdf6d0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9726ca2e-b2a0-4371-96e7-ff9bbdcdf6d0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:25:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:25:30,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:30,706 [286] [DEBUG] [app] Starting request: urn:request:d67be388-389e-4814-a6c8-5c7762a2f576 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:30,706 [286] [DEBUG] [app] Ending request: urn:request:d67be388-389e-4814-a6c8-5c7762a2f576 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d67be388-389e-4814-a6c8-5c7762a2f576', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:25:30,707 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:25:30,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:30,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:30,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:30,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:30,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:30,731 [288] [DEBUG] [app] Ending request: urn:request:1a23d6b3-1f87-4af6-9759-365ae50ea394 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1a23d6b3-1f87-4af6-9759-365ae50ea394', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:30,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:25:30,732 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:25:30,783 [288] [DEBUG] [app] Starting request: urn:request:7dc0cec3-a718-44a8-95ec-8e3ed3f08fb7 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:30,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:30,785 [299] [DEBUG] [app] Starting request: urn:request:e326eb29-aeae-4e48-b528-1b6b1e715d5a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:30,785 [299] [DEBUG] [app] Ending request: urn:request:e326eb29-aeae-4e48-b528-1b6b1e715d5a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e326eb29-aeae-4e48-b528-1b6b1e715d5a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:25:30,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:25:30,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:30,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:30,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:30,790 [285] [DEBUG] [app] Starting request: urn:request:05c3b46b-0fef-46d6-81fc-f49d4e4b6e1a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:30,790 [285] [DEBUG] [app] Ending request: urn:request:05c3b46b-0fef-46d6-81fc-f49d4e4b6e1a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:05c3b46b-0fef-46d6-81fc-f49d4e4b6e1a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:25:30,790 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:25:30,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:30,791 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:30,791 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:30,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:30,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:30,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:30,808 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:30,811 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:30,814 [288] [DEBUG] [app] Ending request: urn:request:7dc0cec3-a718-44a8-95ec-8e3ed3f08fb7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7dc0cec3-a718-44a8-95ec-8e3ed3f08fb7', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:30,814 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:25:30,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.032) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:31,381 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:25:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:25:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:41.530064+00:00 (in 9.999545 seconds) notificationworker stdout | 2025-11-04 09:25:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:41 GMT)" (scheduled at 2025-11-04 09:25:31.530064+00:00) notificationworker stdout | 2025-11-04 09:25:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:25:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 31, 530772), True, datetime.datetime(2025, 11, 4, 9, 25, 31, 530772), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:25:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:25:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:25:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:25:32,178 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:25:32,366 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:25:32,377 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:25:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:25:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:21.434460+00:00 (in 48.998288 seconds) namespacegcworker stdout | 2025-11-04 09:25:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:32 GMT)" (scheduled at 2025-11-04 09:25:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:25:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:25:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:32 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:25:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:25:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:48.625192+00:00 (in 15.001804 seconds) exportactionlogsworker stdout | 2025-11-04 09:25:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:33 GMT)" (scheduled at 2025-11-04 09:25:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:25:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:25:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 33, 623677), True, datetime.datetime(2025, 11, 4, 9, 25, 33, 623677), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:25:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:25:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:25:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:25:33,665 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:25:33,667 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:25:33,760 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:25:34,540 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:25:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:25:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:38.437104+00:00 (in 3.000873 seconds) gcworker stdout | 2025-11-04 09:25:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:05 GMT)" (scheduled at 2025-11-04 09:25:35.435809+00:00) gcworker stdout | 2025-11-04 09:25:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038735436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:25:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:25:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:25:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:25:35,852 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:25:36,224 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:25:36,579 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:25:36,593 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:25:36,593 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:25:36,603 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:25:36,838 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:25:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:25:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:40.124445+00:00 (in 3.001146 seconds) proxycacheblobworker stdout | 2025-11-04 09:25:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:47 GMT)" (scheduled at 2025-11-04 09:25:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:25:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:25:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 37, 123578), True, datetime.datetime(2025, 11, 4, 9, 25, 37, 123578), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:25:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:25:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:25:37,243 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,225 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,231 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,247 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,259 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,280 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,285 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,321 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:25:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:25:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:05.435809+00:00 (in 26.998291 seconds) gcworker stdout | 2025-11-04 09:25:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:08 GMT)" (scheduled at 2025-11-04 09:25:38.437104+00:00) gcworker stdout | 2025-11-04 09:25:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:25:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248278449, None, 1, 0]) gcworker stdout | 2025-11-04 09:25:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:25:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:25:38,471 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:25:38,475 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:25:38,656 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:25:38,662 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:25:38,663 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:25:39,393 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:25:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:25:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:01.922827+00:00 (in 21.997776 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:25:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:39 GMT)" (scheduled at 2025-11-04 09:25:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:25:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:25:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:25:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:25:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:47.122849+00:00 (in 6.997971 seconds) proxycacheblobworker stdout | 2025-11-04 09:25:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:40 GMT)" (scheduled at 2025-11-04 09:25:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:25:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:25:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:25:41,161 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:25:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:25:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:51.530064+00:00 (in 9.999553 seconds) notificationworker stdout | 2025-11-04 09:25:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:51 GMT)" (scheduled at 2025-11-04 09:25:41.530064+00:00) notificationworker stdout | 2025-11-04 09:25:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:25:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 41, 530785), True, datetime.datetime(2025, 11, 4, 9, 25, 41, 530785), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:25:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:25:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:25:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:25:42,610 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:25:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:25:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:57.523915+00:00 (in 14.090593 seconds) repositorygcworker stdout | 2025-11-04 09:25:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:43 GMT)" (scheduled at 2025-11-04 09:25:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:25:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:25:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 43, 433571), True, datetime.datetime(2025, 11, 4, 9, 25, 43, 433571), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:25:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:25:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:25:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:25:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:25:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:59.351216+00:00 (in 14.999624 seconds) quotatotalworker stdout | 2025-11-04 09:25:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:59 GMT)" (scheduled at 2025-11-04 09:25:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:25:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:25:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:25:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:25:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:25:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:25:45,119 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:25:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:25:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:47.479080+00:00 (in 1.998901 seconds) securityworker stdout | 2025-11-04 09:25:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:15 GMT)" (scheduled at 2025-11-04 09:25:45.479769+00:00) securityworker stdout | 2025-11-04 09:25:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:25:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:25:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:25:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 20, 45, 483940), 1, 2]) securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 20, 45, 483940), 1, 2]) securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:25:45,697 [288] [DEBUG] [app] Starting request: urn:request:c7ad7860-0af1-458e-8d11-dc08823ab373 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:45,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:45,701 [292] [DEBUG] [app] Starting request: urn:request:932f09bd-1241-4991-8a84-ea7213c78d87 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:45,702 [292] [DEBUG] [app] Ending request: urn:request:932f09bd-1241-4991-8a84-ea7213c78d87 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:932f09bd-1241-4991-8a84-ea7213c78d87', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:25:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:25:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:45,706 [285] [DEBUG] [app] Starting request: urn:request:8ba19eb3-c56f-47fd-a1e5-19487791827d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:45,706 [285] [DEBUG] [app] Ending request: urn:request:8ba19eb3-c56f-47fd-a1e5-19487791827d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8ba19eb3-c56f-47fd-a1e5-19487791827d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:25:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:25:45,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:45,716 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:45,716 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:45,726 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:45,729 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:45,732 [288] [DEBUG] [app] Ending request: urn:request:c7ad7860-0af1-458e-8d11-dc08823ab373 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c7ad7860-0af1-458e-8d11-dc08823ab373', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:45,732 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:25:45,732 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) gunicorn-web stdout | 2025-11-04 09:25:45,783 [285] [DEBUG] [app] Starting request: urn:request:a1fe6e13-531c-454f-ad07-e4233715dda2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:25:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:25:45,787 [292] [DEBUG] [app] Starting request: urn:request:7828433a-4247-4a42-ad89-b5b8ea4bf0fd (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:25:45,788 [292] [DEBUG] [app] Ending request: urn:request:7828433a-4247-4a42-ad89-b5b8ea4bf0fd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7828433a-4247-4a42-ad89-b5b8ea4bf0fd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:25:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:25:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:25:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:25:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:25:45,792 [288] [DEBUG] [app] Starting request: urn:request:e229a38b-16ba-4cd4-b952-726476de20a6 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:25:45,792 [288] [DEBUG] [app] Ending request: urn:request:e229a38b-16ba-4cd4-b952-726476de20a6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e229a38b-16ba-4cd4-b952-726476de20a6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:25:45,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:25:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:25:45,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:25:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:25:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:25:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:25:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:25:45,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:25:45,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:25:45,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:25:45,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:25:45,816 [285] [DEBUG] [app] Ending request: urn:request:a1fe6e13-531c-454f-ad07-e4233715dda2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a1fe6e13-531c-454f-ad07-e4233715dda2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:25:45,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:25:45,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:25:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:25:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) builder stdout | 2025-11-04 09:25:46,603 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:25:46,619 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:25:46,619 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:25:46,629 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:25:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:25:57.122849+00:00 (in 9.999572 seconds) proxycacheblobworker stdout | 2025-11-04 09:25:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:57 GMT)" (scheduled at 2025-11-04 09:25:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:25:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:25:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 47, 123610), True, datetime.datetime(2025, 11, 4, 9, 25, 47, 123610), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:25:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:25:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:25:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:25:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:25:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:15.479769+00:00 (in 28.000222 seconds) securityworker stdout | 2025-11-04 09:25:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:17 GMT)" (scheduled at 2025-11-04 09:25:47.479080+00:00) securityworker stdout | 2025-11-04 09:25:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:25:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:25:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:25:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:25:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:25:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 20, 47, 483123), 1, 2]) securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:25:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:25:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 20, 47, 483123), 1, 2]) securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:25:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:25:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:25:48,054 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:25:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:25:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:33.622985+00:00 (in 44.997285 seconds) exportactionlogsworker stdout | 2025-11-04 09:25:48,626 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:48 GMT)" (scheduled at 2025-11-04 09:25:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:25:48,626 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:25:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:25:48,713 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:25:50,034 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:25:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:25:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:21.226294+00:00 (in 29.999596 seconds) autopruneworker stdout | 2025-11-04 09:25:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:21 GMT)" (scheduled at 2025-11-04 09:25:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:25:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248291234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:25:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:25:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:25:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:25:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:25:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:01.530064+00:00 (in 9.999526 seconds) notificationworker stdout | 2025-11-04 09:25:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:01 GMT)" (scheduled at 2025-11-04 09:25:51.530064+00:00) notificationworker stdout | 2025-11-04 09:25:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:25:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 51, 530788), True, datetime.datetime(2025, 11, 4, 9, 25, 51, 530788), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:25:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:25:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:25:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:25:52,304 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:25:55,165 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:25:56,630 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:25:56,644 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:25:56,644 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:25:56,654 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:25:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:07.122849+00:00 (in 9.999581 seconds) proxycacheblobworker stdout | 2025-11-04 09:25:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:07 GMT)" (scheduled at 2025-11-04 09:25:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:25:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:25:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 25, 57, 123554), True, datetime.datetime(2025, 11, 4, 9, 25, 57, 123554), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:25:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:09.133645+00:00 (in 12.001440 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:57 GMT)" (scheduled at 2025-11-04 09:25:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:25:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:25:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:25:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:25:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:25:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:25:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:25:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:25:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:25:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:27.222846+00:00 (in 29.999589 seconds) buildlogsarchiver stdout | 2025-11-04 09:25:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:27 GMT)" (scheduled at 2025-11-04 09:25:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:25:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 25, 57, 223456), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:25:57,237 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:25:57,240 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:25:57,241 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:25:57,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:25:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:25:57,387 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:25:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:25:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:43.432867+00:00 (in 45.908225 seconds) repositorygcworker stdout | 2025-11-04 09:25:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:57 GMT)" (scheduled at 2025-11-04 09:25:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:25:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:25:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:26:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:25:58,107 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:25:58,762 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:25:59,144 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:25:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:25:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:14.351216+00:00 (in 14.999616 seconds) quotatotalworker stdout | 2025-11-04 09:25:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:14 GMT)" (scheduled at 2025-11-04 09:25:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:25:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:25:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:25:59,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:25:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:25:59,518 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:59,754 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:59,930 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:25:59,945 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:00,032 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:00,697 [285] [DEBUG] [app] Starting request: urn:request:faa5def1-dde5-48d9-8d33-0581e1247c15 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:00,701 [297] [DEBUG] [app] Starting request: urn:request:a9a0f163-b57b-4e52-b38d-c13db848fd52 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:00,702 [297] [DEBUG] [app] Ending request: urn:request:a9a0f163-b57b-4e52-b38d-c13db848fd52 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a9a0f163-b57b-4e52-b38d-c13db848fd52', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:26:00,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:26:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:00,706 [288] [DEBUG] [app] Starting request: urn:request:779656b3-5a8c-40ce-a232-5167d06801c9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:00,706 [288] [DEBUG] [app] Ending request: urn:request:779656b3-5a8c-40ce-a232-5167d06801c9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:779656b3-5a8c-40ce-a232-5167d06801c9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:26:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:26:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:00,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:00,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:00,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:00,731 [285] [DEBUG] [app] Ending request: urn:request:faa5def1-dde5-48d9-8d33-0581e1247c15 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:faa5def1-dde5-48d9-8d33-0581e1247c15', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:26:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:26:00,784 [285] [DEBUG] [app] Starting request: urn:request:4bd61a98-abf2-4364-b407-50981ceb1bb2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:00,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:00,786 [297] [DEBUG] [app] Starting request: urn:request:b34fe355-6b7d-4170-9598-cf6277ae7f1a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:00,786 [297] [DEBUG] [app] Ending request: urn:request:b34fe355-6b7d-4170-9598-cf6277ae7f1a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b34fe355-6b7d-4170-9598-cf6277ae7f1a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:26:00,786 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:26:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:00,790 [288] [DEBUG] [app] Starting request: urn:request:d21aa289-ee2e-484d-8bb3-a17aad23de7d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:00,790 [288] [DEBUG] [app] Ending request: urn:request:d21aa289-ee2e-484d-8bb3-a17aad23de7d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d21aa289-ee2e-484d-8bb3-a17aad23de7d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:26:00,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:26:00,791 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:00,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:00,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:00,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:00,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:00,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:00,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:00,814 [285] [DEBUG] [app] Ending request: urn:request:4bd61a98-abf2-4364-b407-50981ceb1bb2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4bd61a98-abf2-4364-b407-50981ceb1bb2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:00,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:26:00,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:26:01,390 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:26:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:26:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:11.530064+00:00 (in 9.999558 seconds) notificationworker stdout | 2025-11-04 09:26:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:11 GMT)" (scheduled at 2025-11-04 09:26:01.530064+00:00) notificationworker stdout | 2025-11-04 09:26:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:26:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 1, 530770), True, datetime.datetime(2025, 11, 4, 9, 26, 1, 530770), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:26:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:26:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:26:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:26:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:26:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:39.924652+00:00 (in 38.001444 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:26:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:01 GMT)" (scheduled at 2025-11-04 09:26:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:26:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:26:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 1, 923510), True, datetime.datetime(2025, 11, 4, 9, 26, 1, 923510), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:26:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:26:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:26:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:26:02,187 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:26:02,385 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:26:02,376 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:26:03,674 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:26:03,676 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:26:03,769 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:26:04,550 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:26:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:26:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:08.437104+00:00 (in 3.000887 seconds) gcworker stdout | 2025-11-04 09:26:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:35 GMT)" (scheduled at 2025-11-04 09:26:05.435809+00:00) gcworker stdout | 2025-11-04 09:26:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038765436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:26:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:26:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:26:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:26:05,861 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:26:06,233 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:26:06,655 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:26:06,668 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:26:06,668 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:26:06,678 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:26:06,847 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:26:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:26:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:17.122849+00:00 (in 9.999573 seconds) proxycacheblobworker stdout | 2025-11-04 09:26:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:17 GMT)" (scheduled at 2025-11-04 09:26:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:26:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:26:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 7, 123589), True, datetime.datetime(2025, 11, 4, 9, 26, 7, 123589), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:26:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:26:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:26:07,256 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,240 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,243 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,257 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,274 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,298 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,293 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,333 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:26:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:26:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:35.435809+00:00 (in 26.998312 seconds) gcworker stdout | 2025-11-04 09:26:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:38 GMT)" (scheduled at 2025-11-04 09:26:08.437104+00:00) gcworker stdout | 2025-11-04 09:26:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:26:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248308449, None, 1, 0]) gcworker stdout | 2025-11-04 09:26:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:26:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:26:08,485 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:08,489 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:26:08,673 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:26:08,673 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:26:08,666 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:26:09,134 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:26:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:57.131795+00:00 (in 47.997185 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:26:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:09 GMT)" (scheduled at 2025-11-04 09:26:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:26:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:26:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:26:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:26:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:26:09,402 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:26:11,169 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:26:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:26:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:15.535095+00:00 (in 4.004626 seconds) notificationworker stdout | 2025-11-04 09:26:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:21 GMT)" (scheduled at 2025-11-04 09:26:11.530064+00:00) notificationworker stdout | 2025-11-04 09:26:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:26:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 11, 530789), True, datetime.datetime(2025, 11, 4, 9, 26, 11, 530789), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:26:11,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:26:11,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:26:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:26:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:26:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:12.222828+00:00 (in 59.999554 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:26:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:12 GMT)" (scheduled at 2025-11-04 09:26:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:26:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:26:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:26:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:26:12,625 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:26:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:26:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:29.351216+00:00 (in 14.999606 seconds) quotatotalworker stdout | 2025-11-04 09:26:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:29 GMT)" (scheduled at 2025-11-04 09:26:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:26:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:26:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:26:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:26:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:26:15,128 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:26:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:26:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:17.479080+00:00 (in 1.998855 seconds) securityworker stdout | 2025-11-04 09:26:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:45 GMT)" (scheduled at 2025-11-04 09:26:15.479769+00:00) securityworker stdout | 2025-11-04 09:26:15,481 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:26:15,481 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:26:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:26:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:15,496 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:15,500 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:15,500 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 21, 15, 484373), 1, 2]) securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:15,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 21, 15, 484373), 1, 2]) securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:15,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:15,508 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:15,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:15,508 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:26:15,508 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:26:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:26:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:21.530064+00:00 (in 5.994509 seconds) notificationworker stdout | 2025-11-04 09:26:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:15 GMT)" (scheduled at 2025-11-04 09:26:15.535095+00:00) notificationworker stdout | 2025-11-04 09:26:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:26:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:26:15,697 [287] [DEBUG] [app] Starting request: urn:request:f0275e77-276d-4dfd-bd63-0dabf63285ae (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:15,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:15,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:15,701 [296] [DEBUG] [app] Starting request: urn:request:cefbcc01-697b-4d5e-9ebf-6c3333fcb46f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:15,702 [296] [DEBUG] [app] Ending request: urn:request:cefbcc01-697b-4d5e-9ebf-6c3333fcb46f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cefbcc01-697b-4d5e-9ebf-6c3333fcb46f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:26:15,702 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:26:15,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:15,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:15,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:15,706 [288] [DEBUG] [app] Starting request: urn:request:b435aca0-37fd-454e-8aa1-ea3bb1977b0d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:15,706 [288] [DEBUG] [app] Ending request: urn:request:b435aca0-37fd-454e-8aa1-ea3bb1977b0d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b435aca0-37fd-454e-8aa1-ea3bb1977b0d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:26:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:26:15,707 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:15,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:15,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:15,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:15,716 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:15,716 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:15,726 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:15,729 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:15,732 [287] [DEBUG] [app] Ending request: urn:request:f0275e77-276d-4dfd-bd63-0dabf63285ae (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f0275e77-276d-4dfd-bd63-0dabf63285ae', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:15,733 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:26:15,733 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.037 47 0.037) gunicorn-web stdout | 2025-11-04 09:26:15,784 [288] [DEBUG] [app] Starting request: urn:request:3f6b7f5e-139c-47a8-b28e-7035e60f883d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:15,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:15,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:15,788 [292] [DEBUG] [app] Starting request: urn:request:d35ee2cb-1e82-45e1-81e1-51e167634b18 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:15,788 [292] [DEBUG] [app] Ending request: urn:request:d35ee2cb-1e82-45e1-81e1-51e167634b18 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d35ee2cb-1e82-45e1-81e1-51e167634b18', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:26:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:26:15,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:15,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:15,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:15,792 [285] [DEBUG] [app] Starting request: urn:request:e525e94b-7363-4b13-b217-9aaa344a8028 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:15,792 [285] [DEBUG] [app] Ending request: urn:request:e525e94b-7363-4b13-b217-9aaa344a8028 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e525e94b-7363-4b13-b217-9aaa344a8028', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:26:15,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:26:15,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:15,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:15,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:15,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:15,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:15,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:15,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:15,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:15,817 [288] [DEBUG] [app] Ending request: urn:request:3f6b7f5e-139c-47a8-b28e-7035e60f883d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3f6b7f5e-139c-47a8-b28e-7035e60f883d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:15,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:26:15,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) builder stdout | 2025-11-04 09:26:16,678 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:26:16,692 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:26:16,692 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:26:16,702 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:26:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:27.122849+00:00 (in 9.999627 seconds) proxycacheblobworker stdout | 2025-11-04 09:26:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:27 GMT)" (scheduled at 2025-11-04 09:26:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:26:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:26:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 17, 123453), True, datetime.datetime(2025, 11, 4, 9, 26, 17, 123453), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:26:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:26:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:17,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:26:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:26:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:45.479769+00:00 (in 28.000260 seconds) securityworker stdout | 2025-11-04 09:26:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:47 GMT)" (scheduled at 2025-11-04 09:26:17.479080+00:00) securityworker stdout | 2025-11-04 09:26:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:26:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:26:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:26:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:26:17,495 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:26:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:26:18,062 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:26:18,739 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:26:20,043 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:26:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:26:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:51.226294+00:00 (in 29.999583 seconds) autopruneworker stdout | 2025-11-04 09:26:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:51 GMT)" (scheduled at 2025-11-04 09:26:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:26:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248321234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:26:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:26:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:26:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:26:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:26:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:32.435813+00:00 (in 11.000940 seconds) namespacegcworker stdout | 2025-11-04 09:26:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:21 GMT)" (scheduled at 2025-11-04 09:26:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:26:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:26:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 21, 435168), True, datetime.datetime(2025, 11, 4, 9, 26, 21, 435168), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:26:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:26:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:26:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:26:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:26:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:31.530064+00:00 (in 9.999614 seconds) notificationworker stdout | 2025-11-04 09:26:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:31 GMT)" (scheduled at 2025-11-04 09:26:21.530064+00:00) notificationworker stdout | 2025-11-04 09:26:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:26:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 21, 530751), True, datetime.datetime(2025, 11, 4, 9, 26, 21, 530751), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:26:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:26:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:26:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:26:22,313 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:26:25,174 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:26:26,703 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:26:26,718 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:26:26,718 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:26:26,728 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:26:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:37.122849+00:00 (in 9.999588 seconds) proxycacheblobworker stdout | 2025-11-04 09:26:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:37 GMT)" (scheduled at 2025-11-04 09:26:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:26:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:26:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 27, 123536), True, datetime.datetime(2025, 11, 4, 9, 26, 27, 123536), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:26:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:26:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:26:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:26:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:57.222846+00:00 (in 29.999598 seconds) buildlogsarchiver stdout | 2025-11-04 09:26:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:57 GMT)" (scheduled at 2025-11-04 09:26:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:26:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 26, 27, 223463), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:26:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:26:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:26:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:26:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:26:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:26:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:26:27,396 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:26:28,116 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:26:28,770 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:26:29,153 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:26:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:26:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:44.351216+00:00 (in 14.999623 seconds) quotatotalworker stdout | 2025-11-04 09:26:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:44 GMT)" (scheduled at 2025-11-04 09:26:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:26:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:26:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:26:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:26:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:26:29,528 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:29,766 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:29,943 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:29,958 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:30,049 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:30,697 [285] [DEBUG] [app] Starting request: urn:request:10d5496d-e365-4a52-ba6e-391bb1c59398 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:30,701 [298] [DEBUG] [app] Starting request: urn:request:76d4e86e-6eb5-4be9-97ad-86bf6bfe62cb (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:30,702 [298] [DEBUG] [app] Ending request: urn:request:76d4e86e-6eb5-4be9-97ad-86bf6bfe62cb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:76d4e86e-6eb5-4be9-97ad-86bf6bfe62cb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:26:30,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:26:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:30,706 [288] [DEBUG] [app] Starting request: urn:request:b723c813-48d1-4737-b4b9-9175d6a57935 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:30,706 [288] [DEBUG] [app] Ending request: urn:request:b723c813-48d1-4737-b4b9-9175d6a57935 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b723c813-48d1-4737-b4b9-9175d6a57935', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:26:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:26:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:30,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:30,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:30,731 [285] [DEBUG] [app] Ending request: urn:request:10d5496d-e365-4a52-ba6e-391bb1c59398 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:10d5496d-e365-4a52-ba6e-391bb1c59398', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:30,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:26:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:26:30,784 [285] [DEBUG] [app] Starting request: urn:request:38844aa2-f72e-43bd-8754-075ba84ad813 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:30,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:30,785 [298] [DEBUG] [app] Starting request: urn:request:edfd2605-3695-4f51-b7a3-9f2ba25dafa1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:30,786 [298] [DEBUG] [app] Ending request: urn:request:edfd2605-3695-4f51-b7a3-9f2ba25dafa1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:edfd2605-3695-4f51-b7a3-9f2ba25dafa1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:26:30,786 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:26:30,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:30,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:30,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:30,790 [287] [DEBUG] [app] Starting request: urn:request:1edfe66d-0134-484e-82e9-08739f4a75b1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:30,790 [287] [DEBUG] [app] Ending request: urn:request:1edfe66d-0134-484e-82e9-08739f4a75b1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1edfe66d-0134-484e-82e9-08739f4a75b1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:26:30,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:26:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:30,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:30,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:30,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:30,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:30,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:30,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:30,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:30,814 [285] [DEBUG] [app] Ending request: urn:request:38844aa2-f72e-43bd-8754-075ba84ad813 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:38844aa2-f72e-43bd-8754-075ba84ad813', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:30,814 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) gunicorn-web stdout | 2025-11-04 09:26:30,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:26:31,398 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:26:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:26:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:41.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:26:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:41 GMT)" (scheduled at 2025-11-04 09:26:31.530064+00:00) notificationworker stdout | 2025-11-04 09:26:31,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:26:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 31, 531098), True, datetime.datetime(2025, 11, 4, 9, 26, 31, 531098), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:26:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:26:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:26:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:26:32,196 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:26:32,394 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:26:32,396 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:26:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:26:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:21.434460+00:00 (in 48.998226 seconds) namespacegcworker stdout | 2025-11-04 09:26:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:32 GMT)" (scheduled at 2025-11-04 09:26:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:26:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:26:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:32 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:26:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:26:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:48.625192+00:00 (in 15.001785 seconds) exportactionlogsworker stdout | 2025-11-04 09:26:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:33 GMT)" (scheduled at 2025-11-04 09:26:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:26:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:26:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 33, 623691), True, datetime.datetime(2025, 11, 4, 9, 26, 33, 623691), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:26:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:26:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:26:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:26:33,683 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:26:33,685 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:26:33,788 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:26:34,559 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:26:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:26:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:38.437104+00:00 (in 3.000855 seconds) gcworker stdout | 2025-11-04 09:26:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:05 GMT)" (scheduled at 2025-11-04 09:26:35.435809+00:00) gcworker stdout | 2025-11-04 09:26:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038795436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:26:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:26:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:26:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:26:35,870 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:26:36,242 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:26:36,728 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:26:36,742 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:26:36,742 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:26:36,752 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:26:36,856 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:26:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:26:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:40.124445+00:00 (in 3.001190 seconds) proxycacheblobworker stdout | 2025-11-04 09:26:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:47 GMT)" (scheduled at 2025-11-04 09:26:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:26:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:26:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 37, 123544), True, datetime.datetime(2025, 11, 4, 9, 26, 37, 123544), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:26:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:26:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:26:37,266 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,254 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,255 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,268 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,287 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,312 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,320 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,348 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:26:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:26:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:05.435809+00:00 (in 26.998293 seconds) gcworker stdout | 2025-11-04 09:26:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:08 GMT)" (scheduled at 2025-11-04 09:26:38.437104+00:00) gcworker stdout | 2025-11-04 09:26:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:26:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248338449, None, 1, 0]) gcworker stdout | 2025-11-04 09:26:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:26:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:26:38,500 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:26:38,502 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:26:38,683 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:26:38,684 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:26:38,688 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:26:39,411 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:26:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:26:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:01.922827+00:00 (in 21.997731 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:26:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:39 GMT)" (scheduled at 2025-11-04 09:26:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:26:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:26:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:26:40,125 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:26:40,125 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:47.122849+00:00 (in 6.997551 seconds) proxycacheblobworker stdout | 2025-11-04 09:26:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:40 GMT)" (scheduled at 2025-11-04 09:26:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:26:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:26:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:26:41,178 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:26:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:26:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:51.530064+00:00 (in 9.999611 seconds) notificationworker stdout | 2025-11-04 09:26:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:51 GMT)" (scheduled at 2025-11-04 09:26:41.530064+00:00) notificationworker stdout | 2025-11-04 09:26:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:26:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 41, 530737), True, datetime.datetime(2025, 11, 4, 9, 26, 41, 530737), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:26:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:26:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:26:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:26:42,633 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:26:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:26:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:57.523915+00:00 (in 14.090644 seconds) repositorygcworker stdout | 2025-11-04 09:26:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:43 GMT)" (scheduled at 2025-11-04 09:26:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:26:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:26:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 43, 433516), True, datetime.datetime(2025, 11, 4, 9, 26, 43, 433516), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:26:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:26:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:26:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:26:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:26:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:59.351216+00:00 (in 14.999617 seconds) quotatotalworker stdout | 2025-11-04 09:26:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:59 GMT)" (scheduled at 2025-11-04 09:26:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:26:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:26:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:26:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:26:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:26:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:26:45,137 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:26:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:26:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:47.479080+00:00 (in 1.998888 seconds) securityworker stdout | 2025-11-04 09:26:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:15 GMT)" (scheduled at 2025-11-04 09:26:45.479769+00:00) securityworker stdout | 2025-11-04 09:26:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:26:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:26:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:26:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 21, 45, 483894), 1, 2]) securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 21, 45, 483894), 1, 2]) securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:26:45,696 [288] [DEBUG] [app] Starting request: urn:request:a578de3e-619a-407c-bba3-a564e69d0e21 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:45,701 [292] [DEBUG] [app] Starting request: urn:request:edd071a9-589d-42a1-a1e4-27074ae56dba (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:45,701 [292] [DEBUG] [app] Ending request: urn:request:edd071a9-589d-42a1-a1e4-27074ae56dba (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:edd071a9-589d-42a1-a1e4-27074ae56dba', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:26:45,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:26:45,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:45,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:45,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:45,705 [285] [DEBUG] [app] Starting request: urn:request:dd6363c4-2508-43dd-b9f6-0d53d87ee0c7 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:45,705 [285] [DEBUG] [app] Ending request: urn:request:dd6363c4-2508-43dd-b9f6-0d53d87ee0c7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dd6363c4-2508-43dd-b9f6-0d53d87ee0c7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:26:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:26:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:45,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:45,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:45,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:45,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:45,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:45,730 [288] [DEBUG] [app] Ending request: urn:request:a578de3e-619a-407c-bba3-a564e69d0e21 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a578de3e-619a-407c-bba3-a564e69d0e21', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:45,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:26:45,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:26:45,784 [288] [DEBUG] [app] Starting request: urn:request:d85c547d-3b3d-4708-a956-b2fe09a6f835 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:26:45,785 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:26:45,786 [299] [DEBUG] [app] Starting request: urn:request:6bcbfcfe-bf6e-49ce-8a5b-65410ccd3450 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:26:45,786 [299] [DEBUG] [app] Ending request: urn:request:6bcbfcfe-bf6e-49ce-8a5b-65410ccd3450 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6bcbfcfe-bf6e-49ce-8a5b-65410ccd3450', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:26:45,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:26:45,787 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:45,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:26:45,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:26:45,790 [285] [DEBUG] [app] Starting request: urn:request:64063f9c-b3c4-424e-ad67-1a7c649ace0c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:26:45,790 [285] [DEBUG] [app] Ending request: urn:request:64063f9c-b3c4-424e-ad67-1a7c649ace0c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:64063f9c-b3c4-424e-ad67-1a7c649ace0c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:26:45,791 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:26:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:26:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.000 159 0.001) gunicorn-web stdout | 2025-11-04 09:26:45,791 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:26:45,791 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:26:45,791 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:26:45,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:26:45,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:26:45,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:26:45,808 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:26:45,811 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:26:45,814 [288] [DEBUG] [app] Ending request: urn:request:d85c547d-3b3d-4708-a956-b2fe09a6f835 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d85c547d-3b3d-4708-a956-b2fe09a6f835', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:26:45,814 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:26:45,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:26:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:26:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) builder stdout | 2025-11-04 09:26:46,752 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:26:46,765 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:26:46,765 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:26:46,775 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:26:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:57.122849+00:00 (in 9.999609 seconds) proxycacheblobworker stdout | 2025-11-04 09:26:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:57 GMT)" (scheduled at 2025-11-04 09:26:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:26:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:26:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 47, 123504), True, datetime.datetime(2025, 11, 4, 9, 26, 47, 123504), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:26:47,138 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:26:47,138 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:47,138 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:26:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:26:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:26:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:15.479769+00:00 (in 28.000247 seconds) securityworker stdout | 2025-11-04 09:26:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:17 GMT)" (scheduled at 2025-11-04 09:26:47.479080+00:00) securityworker stdout | 2025-11-04 09:26:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:26:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:26:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:26:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:26:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 21, 47, 482969), 1, 2]) securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:26:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:26:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 21, 47, 482969), 1, 2]) securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:26:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:26:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:26:48,081 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:26:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:26:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:26:58.624878+00:00 (in 9.999182 seconds) exportactionlogsworker stdout | 2025-11-04 09:26:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:48 GMT)" (scheduled at 2025-11-04 09:26:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:26:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:26:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:26:48,748 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:26:50,052 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:26:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:26:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:21.226294+00:00 (in 29.999584 seconds) autopruneworker stdout | 2025-11-04 09:26:51,227 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:21 GMT)" (scheduled at 2025-11-04 09:26:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:26:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248351235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:26:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:26:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:26:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:26:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:26:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:01.530064+00:00 (in 9.999613 seconds) notificationworker stdout | 2025-11-04 09:26:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:01 GMT)" (scheduled at 2025-11-04 09:26:51.530064+00:00) notificationworker stdout | 2025-11-04 09:26:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:26:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 51, 530723), True, datetime.datetime(2025, 11, 4, 9, 26, 51, 530723), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:26:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:26:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:26:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:26:52,333 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:26:55,182 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:26:56,775 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:26:56,790 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:26:56,790 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:26:56,800 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:26:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:07.122849+00:00 (in 9.999594 seconds) proxycacheblobworker stdout | 2025-11-04 09:26:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:07 GMT)" (scheduled at 2025-11-04 09:26:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:26:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:26:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 26, 57, 123524), True, datetime.datetime(2025, 11, 4, 9, 26, 57, 123524), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:26:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:26:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:09.133645+00:00 (in 12.001411 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:26:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:57 GMT)" (scheduled at 2025-11-04 09:26:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:26:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:26:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:26:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:26:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:26:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:26:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:26:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:26:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:26:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:27.222846+00:00 (in 29.999601 seconds) buildlogsarchiver stdout | 2025-11-04 09:26:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:27 GMT)" (scheduled at 2025-11-04 09:26:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:26:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 26, 57, 223469), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:26:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:26:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:26:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:26:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:26:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:26:57,405 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:26:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:26:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:43.432867+00:00 (in 45.908511 seconds) repositorygcworker stdout | 2025-11-04 09:26:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:57 GMT)" (scheduled at 2025-11-04 09:26:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:26:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:26:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:27:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:26:58,125 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:26:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:26:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:33.622985+00:00 (in 34.997683 seconds) exportactionlogsworker stdout | 2025-11-04 09:26:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:31:58 GMT)" (scheduled at 2025-11-04 09:26:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:26:58,626 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 26, 58, 625569), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:26:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 26, 58, 625569), True, datetime.datetime(2025, 11, 4, 9, 26, 58, 625569), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:26:58,641 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 26, 58, 625569), True, datetime.datetime(2025, 11, 4, 9, 26, 58, 625569), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 26, 58, 625569), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:26:58,644 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:26:58,645 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:31:58 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:26:58,780 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:26:59,161 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:26:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:26:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:14.351216+00:00 (in 14.999541 seconds) quotatotalworker stdout | 2025-11-04 09:26:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:14 GMT)" (scheduled at 2025-11-04 09:26:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:26:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:26:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:26:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:26:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:26:59,539 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:59,777 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:59,957 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:26:59,972 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:00,064 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:00,697 [286] [DEBUG] [app] Starting request: urn:request:cc9128d0-ae1b-4034-a71c-28a9c022f10f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:00,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:00,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:00,702 [296] [DEBUG] [app] Starting request: urn:request:179d9b66-c5f8-4fa3-a5b0-883c98274db7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:00,702 [296] [DEBUG] [app] Ending request: urn:request:179d9b66-c5f8-4fa3-a5b0-883c98274db7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:179d9b66-c5f8-4fa3-a5b0-883c98274db7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:27:00,703 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:27:00,703 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:00,704 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:00,706 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:00,707 [285] [DEBUG] [app] Starting request: urn:request:746f7166-15a0-4b57-a013-7eef2d29825e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:00,707 [285] [DEBUG] [app] Ending request: urn:request:746f7166-15a0-4b57-a013-7eef2d29825e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:746f7166-15a0-4b57-a013-7eef2d29825e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:27:00,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:27:00,707 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:00,708 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:00,708 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:00,708 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:00,716 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:00,716 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:00,725 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:00,729 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:00,732 [286] [DEBUG] [app] Ending request: urn:request:cc9128d0-ae1b-4034-a71c-28a9c022f10f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:cc9128d0-ae1b-4034-a71c-28a9c022f10f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:00,732 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:27:00,732 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.036) gunicorn-web stdout | 2025-11-04 09:27:00,784 [287] [DEBUG] [app] Starting request: urn:request:1e428005-5f5b-4c97-b07c-1b0a558917dc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:00,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:00,788 [299] [DEBUG] [app] Starting request: urn:request:3c14e6f3-ab47-480d-a627-a1000ef8ef60 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:00,788 [299] [DEBUG] [app] Ending request: urn:request:3c14e6f3-ab47-480d-a627-a1000ef8ef60 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3c14e6f3-ab47-480d-a627-a1000ef8ef60', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:27:00,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:27:00,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:00,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:00,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:00,792 [285] [DEBUG] [app] Starting request: urn:request:4ccd6912-e99c-4aaf-9db9-c6d2bb4ad462 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:00,792 [285] [DEBUG] [app] Ending request: urn:request:4ccd6912-e99c-4aaf-9db9-c6d2bb4ad462 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4ccd6912-e99c-4aaf-9db9-c6d2bb4ad462', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:27:00,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:00,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:00,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:00,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:00,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:00,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:00,814 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:00,818 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:00,821 [287] [DEBUG] [app] Ending request: urn:request:1e428005-5f5b-4c97-b07c-1b0a558917dc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1e428005-5f5b-4c97-b07c-1b0a558917dc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:00,821 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.038 47 0.038) gunicorn-web stdout | 2025-11-04 09:27:00,821 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:27:01,407 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:27:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:27:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:11.530064+00:00 (in 9.999583 seconds) notificationworker stdout | 2025-11-04 09:27:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:11 GMT)" (scheduled at 2025-11-04 09:27:01.530064+00:00) notificationworker stdout | 2025-11-04 09:27:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:27:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 1, 530752), True, datetime.datetime(2025, 11, 4, 9, 27, 1, 530752), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:27:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:27:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:27:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:27:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:27:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:21.924304+00:00 (in 20.001084 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:27:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:01 GMT)" (scheduled at 2025-11-04 09:27:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:27:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:27:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 1, 923477), True, datetime.datetime(2025, 11, 4, 9, 27, 1, 923477), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:27:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:27:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:27:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:27:02,205 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:27:02,403 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:27:02,404 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:27:03,691 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:27:03,693 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:27:03,797 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:27:04,569 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:27:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:27:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:08.437104+00:00 (in 3.000901 seconds) gcworker stdout | 2025-11-04 09:27:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:35 GMT)" (scheduled at 2025-11-04 09:27:05.435809+00:00) gcworker stdout | 2025-11-04 09:27:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038825436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:27:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:27:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:27:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:27:05,879 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:27:06,251 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:27:06,800 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:27:06,813 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:27:06,813 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:27:06,823 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:27:06,866 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:27:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:27:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:17.122849+00:00 (in 9.999584 seconds) proxycacheblobworker stdout | 2025-11-04 09:27:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:17 GMT)" (scheduled at 2025-11-04 09:27:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:27:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:27:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 7, 123568), True, datetime.datetime(2025, 11, 4, 9, 27, 7, 123568), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:27:07,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:27:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:27:07,286 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,267 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,267 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,278 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,300 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,324 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,334 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,362 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:27:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:27:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:35.435809+00:00 (in 26.998299 seconds) gcworker stdout | 2025-11-04 09:27:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:38 GMT)" (scheduled at 2025-11-04 09:27:08.437104+00:00) gcworker stdout | 2025-11-04 09:27:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:27:08,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248368449, None, 1, 0]) gcworker stdout | 2025-11-04 09:27:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:27:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:27:08,513 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:08,520 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:27:08,694 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:27:08,694 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:27:08,698 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:27:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:27:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:57.131795+00:00 (in 47.997705 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:27:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:09 GMT)" (scheduled at 2025-11-04 09:27:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:27:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:27:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:27:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:27:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:27:09,420 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:27:11,197 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:27:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:27:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:15.535095+00:00 (in 4.004638 seconds) notificationworker stdout | 2025-11-04 09:27:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:21 GMT)" (scheduled at 2025-11-04 09:27:11.530064+00:00) notificationworker stdout | 2025-11-04 09:27:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:27:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 11, 530781), True, datetime.datetime(2025, 11, 4, 9, 27, 11, 530781), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:27:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:27:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:27:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:27:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:27:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:12.222828+00:00 (in 59.999581 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:27:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:12 GMT)" (scheduled at 2025-11-04 09:27:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:27:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:27:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:27:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:27:12,642 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:27:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:27:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:29.351216+00:00 (in 14.999614 seconds) quotatotalworker stdout | 2025-11-04 09:27:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:29 GMT)" (scheduled at 2025-11-04 09:27:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:27:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:27:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:27:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:27:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:27:15,146 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:27:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:27:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:17.479080+00:00 (in 1.998922 seconds) securityworker stdout | 2025-11-04 09:27:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:45 GMT)" (scheduled at 2025-11-04 09:27:15.479769+00:00) securityworker stdout | 2025-11-04 09:27:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:27:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:27:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:27:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:27:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:15,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:15,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 22, 15, 483588), 1, 2]) securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:15,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 22, 15, 483588), 1, 2]) securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:27:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:27:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:21.530064+00:00 (in 5.994546 seconds) notificationworker stdout | 2025-11-04 09:27:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:15 GMT)" (scheduled at 2025-11-04 09:27:15.535095+00:00) notificationworker stdout | 2025-11-04 09:27:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:27:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:27:15,697 [287] [DEBUG] [app] Starting request: urn:request:7e761dde-267e-46e3-8809-17f6f64f72d2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:15,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:15,701 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:15,702 [299] [DEBUG] [app] Starting request: urn:request:bfc577e6-19e4-49e9-8baf-86f022339d73 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:15,702 [299] [DEBUG] [app] Ending request: urn:request:bfc577e6-19e4-49e9-8baf-86f022339d73 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bfc577e6-19e4-49e9-8baf-86f022339d73', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:27:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:27:15,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:15,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:15,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:15,706 [285] [DEBUG] [app] Starting request: urn:request:88c4fd33-885c-4245-accf-8acb649b94b0 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:15,706 [285] [DEBUG] [app] Ending request: urn:request:88c4fd33-885c-4245-accf-8acb649b94b0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:88c4fd33-885c-4245-accf-8acb649b94b0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:27:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:27:15,707 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:15,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:15,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:15,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:15,715 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:15,715 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:15,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:15,728 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:15,731 [287] [DEBUG] [app] Ending request: urn:request:7e761dde-267e-46e3-8809-17f6f64f72d2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7e761dde-267e-46e3-8809-17f6f64f72d2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:15,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:27:15,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:27:15,784 [285] [DEBUG] [app] Starting request: urn:request:208d2d77-2d89-46f6-9062-c209340cd34a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:15,788 [299] [DEBUG] [app] Starting request: urn:request:07ad1330-8661-45aa-9704-7b58c19ee83a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:15,788 [299] [DEBUG] [app] Ending request: urn:request:07ad1330-8661-45aa-9704-7b58c19ee83a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:07ad1330-8661-45aa-9704-7b58c19ee83a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:27:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:27:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:15,792 [287] [DEBUG] [app] Starting request: urn:request:a3013052-4f10-4cc6-83f7-bfcee9485861 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:15,792 [287] [DEBUG] [app] Ending request: urn:request:a3013052-4f10-4cc6-83f7-bfcee9485861 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a3013052-4f10-4cc6-83f7-bfcee9485861', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:27:15,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:15,817 [285] [DEBUG] [app] Ending request: urn:request:208d2d77-2d89-46f6-9062-c209340cd34a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:208d2d77-2d89-46f6-9062-c209340cd34a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:27:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) builder stdout | 2025-11-04 09:27:16,824 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:27:16,837 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:27:16,838 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:27:16,847 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:27:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:27.122849+00:00 (in 9.999609 seconds) proxycacheblobworker stdout | 2025-11-04 09:27:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:27 GMT)" (scheduled at 2025-11-04 09:27:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:27:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:27:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 17, 123522), True, datetime.datetime(2025, 11, 4, 9, 27, 17, 123522), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:27:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:27:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:27:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:27:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:45.479769+00:00 (in 28.000230 seconds) securityworker stdout | 2025-11-04 09:27:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:47 GMT)" (scheduled at 2025-11-04 09:27:17.479080+00:00) securityworker stdout | 2025-11-04 09:27:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:27:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:27:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:27:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:27:17,496 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:27:17,497 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:27:18,090 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:27:18,756 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:27:20,061 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:27:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:27:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:51.226294+00:00 (in 29.999567 seconds) autopruneworker stdout | 2025-11-04 09:27:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:51 GMT)" (scheduled at 2025-11-04 09:27:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:27:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248381234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:27:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:27:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:27:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:27:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:27:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:32.435813+00:00 (in 11.000865 seconds) namespacegcworker stdout | 2025-11-04 09:27:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:21 GMT)" (scheduled at 2025-11-04 09:27:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:27:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:27:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 21, 435284), True, datetime.datetime(2025, 11, 4, 9, 27, 21, 435284), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:27:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:27:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:27:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:27:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:27:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:31.530064+00:00 (in 9.999654 seconds) notificationworker stdout | 2025-11-04 09:27:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:31 GMT)" (scheduled at 2025-11-04 09:27:21.530064+00:00) notificationworker stdout | 2025-11-04 09:27:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:27:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 21, 530703), True, datetime.datetime(2025, 11, 4, 9, 27, 21, 530703), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:27:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:27:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:27:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:31 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:27:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:27:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:39.924652+00:00 (in 17.999947 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:27:21,924 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:32:21 GMT)" (scheduled at 2025-11-04 09:27:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:27:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 27, 21, 924957), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:27:21,937 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 27, 21, 924957), True, datetime.datetime(2025, 11, 4, 9, 27, 21, 924957), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:27:21,941 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 27, 21, 924957), True, datetime.datetime(2025, 11, 4, 9, 27, 21, 924957), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 27, 21, 924957), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:27:21,944 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:27:21,944 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:32:21 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:27:22,342 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:27:25,192 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:27:26,848 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:27:26,862 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:27:26,862 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:27:26,872 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:27:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:37.122849+00:00 (in 9.999620 seconds) proxycacheblobworker stdout | 2025-11-04 09:27:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:37 GMT)" (scheduled at 2025-11-04 09:27:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:27:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:27:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 27, 123460), True, datetime.datetime(2025, 11, 4, 9, 27, 27, 123460), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:27:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:27:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:27:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:27:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:57.222846+00:00 (in 29.999550 seconds) buildlogsarchiver stdout | 2025-11-04 09:27:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:57 GMT)" (scheduled at 2025-11-04 09:27:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:27:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 27, 27, 223570), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:27:27,237 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:27:27,240 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:27:27,241 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:27:27,247 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:27:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:27:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:27:27,425 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:27:28,134 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:27:28,788 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:27:29,170 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:27:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:27:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:44.351216+00:00 (in 14.999594 seconds) quotatotalworker stdout | 2025-11-04 09:27:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:44 GMT)" (scheduled at 2025-11-04 09:27:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:27:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:27:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:27:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:27:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:27:29,549 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:29,788 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:29,970 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:29,985 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:30,077 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:30,697 [285] [DEBUG] [app] Starting request: urn:request:03b58752-514f-4a73-8558-3e2ae046e87a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:30,701 [299] [DEBUG] [app] Starting request: urn:request:06877bb5-553f-47e5-a93a-777a87b5a7fc (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:30,702 [299] [DEBUG] [app] Ending request: urn:request:06877bb5-553f-47e5-a93a-777a87b5a7fc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:06877bb5-553f-47e5-a93a-777a87b5a7fc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:27:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:27:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:30,706 [288] [DEBUG] [app] Starting request: urn:request:8f9e80c5-d603-4989-ba44-9170933b539b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:30,706 [288] [DEBUG] [app] Ending request: urn:request:8f9e80c5-d603-4989-ba44-9170933b539b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8f9e80c5-d603-4989-ba44-9170933b539b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:27:30,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:30,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:30,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:30,730 [285] [DEBUG] [app] Ending request: urn:request:03b58752-514f-4a73-8558-3e2ae046e87a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:03b58752-514f-4a73-8558-3e2ae046e87a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:30,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:27:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:27:30,784 [285] [DEBUG] [app] Starting request: urn:request:e0db9c2e-f4bf-432b-9f15-dc3394fcbc4d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:30,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:30,785 [299] [DEBUG] [app] Starting request: urn:request:c9dc9b32-d0f7-4a9f-a375-c0b56d667361 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:30,786 [299] [DEBUG] [app] Ending request: urn:request:c9dc9b32-d0f7-4a9f-a375-c0b56d667361 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c9dc9b32-d0f7-4a9f-a375-c0b56d667361', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:27:30,786 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:27:30,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:30,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:30,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:30,790 [288] [DEBUG] [app] Starting request: urn:request:2cdccc72-9116-4968-9435-e95632a86ae6 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:30,790 [288] [DEBUG] [app] Ending request: urn:request:2cdccc72-9116-4968-9435-e95632a86ae6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2cdccc72-9116-4968-9435-e95632a86ae6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:27:30,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:30,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:30,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:30,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:30,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:30,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:30,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:30,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:30,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:30,813 [285] [DEBUG] [app] Ending request: urn:request:e0db9c2e-f4bf-432b-9f15-dc3394fcbc4d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e0db9c2e-f4bf-432b-9f15-dc3394fcbc4d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:30,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:27:30,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.030) manifestsubjectbackfillworker stdout | 2025-11-04 09:27:31,427 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:27:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:27:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:41.530064+00:00 (in 9.999591 seconds) notificationworker stdout | 2025-11-04 09:27:31,531 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:41 GMT)" (scheduled at 2025-11-04 09:27:31.530064+00:00) notificationworker stdout | 2025-11-04 09:27:31,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:27:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 31, 531268), True, datetime.datetime(2025, 11, 4, 9, 27, 31, 531268), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:27:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:27:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:27:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:27:32,213 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:27:32,411 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:27:32,413 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:27:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:27:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:08.435432+00:00 (in 35.999215 seconds) namespacegcworker stdout | 2025-11-04 09:27:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:32 GMT)" (scheduled at 2025-11-04 09:27:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:27:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:27:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:32 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:27:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:27:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:48.625192+00:00 (in 15.001802 seconds) exportactionlogsworker stdout | 2025-11-04 09:27:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:33 GMT)" (scheduled at 2025-11-04 09:27:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:27:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:27:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 33, 623677), True, datetime.datetime(2025, 11, 4, 9, 27, 33, 623677), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:27:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:27:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:27:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:27:33,700 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:27:33,701 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:27:33,806 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:27:34,578 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:27:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:27:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:38.437104+00:00 (in 3.000906 seconds) gcworker stdout | 2025-11-04 09:27:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:05 GMT)" (scheduled at 2025-11-04 09:27:35.435809+00:00) gcworker stdout | 2025-11-04 09:27:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038855436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:27:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:27:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:27:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:27:35,888 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:27:36,260 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:27:36,872 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue securityworker stdout | 2025-11-04 09:27:36,875 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:27:36,886 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:27:36,886 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:27:36,896 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:27:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:40.124445+00:00 (in 3.001168 seconds) proxycacheblobworker stdout | 2025-11-04 09:27:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:47 GMT)" (scheduled at 2025-11-04 09:27:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:27:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:27:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 37, 123554), True, datetime.datetime(2025, 11, 4, 9, 27, 37, 123554), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:27:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:27:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:27:37,295 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,278 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,279 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,292 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,312 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,334 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,347 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,375 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:27:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:27:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:05.435809+00:00 (in 26.998297 seconds) gcworker stdout | 2025-11-04 09:27:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:08 GMT)" (scheduled at 2025-11-04 09:27:38.437104+00:00) gcworker stdout | 2025-11-04 09:27:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:27:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248398449, None, 1, 0]) gcworker stdout | 2025-11-04 09:27:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:27:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:27:38,530 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:27:38,531 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:27:38,704 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:27:38,704 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:27:38,708 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:27:39,429 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:27:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:27:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:01.922827+00:00 (in 21.997760 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:27:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:39 GMT)" (scheduled at 2025-11-04 09:27:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:27:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:27:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:27:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:27:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:47.122849+00:00 (in 6.997995 seconds) proxycacheblobworker stdout | 2025-11-04 09:27:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:40 GMT)" (scheduled at 2025-11-04 09:27:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:27:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:27:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:27:41,206 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:27:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:27:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:51.530064+00:00 (in 9.999542 seconds) notificationworker stdout | 2025-11-04 09:27:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:51 GMT)" (scheduled at 2025-11-04 09:27:41.530064+00:00) notificationworker stdout | 2025-11-04 09:27:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:27:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 41, 530781), True, datetime.datetime(2025, 11, 4, 9, 27, 41, 530781), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:27:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:27:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:27:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:27:42,651 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:27:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:27:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:57.523915+00:00 (in 14.090634 seconds) repositorygcworker stdout | 2025-11-04 09:27:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:43 GMT)" (scheduled at 2025-11-04 09:27:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:27:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:27:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 43, 433533), True, datetime.datetime(2025, 11, 4, 9, 27, 43, 433533), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:27:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:27:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:27:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:27:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:27:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:59.351216+00:00 (in 14.999603 seconds) quotatotalworker stdout | 2025-11-04 09:27:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:59 GMT)" (scheduled at 2025-11-04 09:27:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:27:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:27:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:27:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:27:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:27:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:27:45,155 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:27:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:27:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:47.479080+00:00 (in 1.998901 seconds) securityworker stdout | 2025-11-04 09:27:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:15 GMT)" (scheduled at 2025-11-04 09:27:45.479769+00:00) securityworker stdout | 2025-11-04 09:27:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:27:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:27:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:27:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:27:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 22, 45, 483560), 1, 2]) securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:45,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 22, 45, 483560), 1, 2]) securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:27:45,697 [285] [DEBUG] [app] Starting request: urn:request:025635ab-36da-4a8c-ad58-241c61a6278e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:45,701 [299] [DEBUG] [app] Starting request: urn:request:dc0c88ea-3fb8-4912-bf67-cb8ae4216444 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:45,701 [299] [DEBUG] [app] Ending request: urn:request:dc0c88ea-3fb8-4912-bf67-cb8ae4216444 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:dc0c88ea-3fb8-4912-bf67-cb8ae4216444', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:27:45,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:27:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:45,705 [288] [DEBUG] [app] Starting request: urn:request:9c87a800-3e33-4344-9170-2be00767e50c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:45,706 [288] [DEBUG] [app] Ending request: urn:request:9c87a800-3e33-4344-9170-2be00767e50c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9c87a800-3e33-4344-9170-2be00767e50c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:27:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:45,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:45,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:45,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:45,730 [285] [DEBUG] [app] Ending request: urn:request:025635ab-36da-4a8c-ad58-241c61a6278e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:025635ab-36da-4a8c-ad58-241c61a6278e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:45,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:27:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:27:45,784 [286] [DEBUG] [app] Starting request: urn:request:63dfbbc5-1785-4bb5-95d9-31e9c86fbada (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:27:45,785 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:45,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:27:45,788 [292] [DEBUG] [app] Starting request: urn:request:6eaa3a9a-8fcc-471b-8674-2c5f2b2232d0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:27:45,788 [292] [DEBUG] [app] Ending request: urn:request:6eaa3a9a-8fcc-471b-8674-2c5f2b2232d0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6eaa3a9a-8fcc-471b-8674-2c5f2b2232d0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:27:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:27:45,789 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:45,790 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:27:45,792 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:27:45,793 [285] [DEBUG] [app] Starting request: urn:request:e57eeef5-ff13-4b37-8fbe-c0c0b62a0b88 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:27:45,793 [285] [DEBUG] [app] Ending request: urn:request:e57eeef5-ff13-4b37-8fbe-c0c0b62a0b88 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e57eeef5-ff13-4b37-8fbe-c0c0b62a0b88', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:27:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:27:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:27:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:27:45,793 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:27:45,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:27:45,794 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:27:45,794 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:27:45,801 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:27:45,801 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:27:45,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:27:45,814 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:27:45,817 [286] [DEBUG] [app] Ending request: urn:request:63dfbbc5-1785-4bb5-95d9-31e9c86fbada (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:63dfbbc5-1785-4bb5-95d9-31e9c86fbada', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:27:45,817 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:27:45,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:27:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:27:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) builder stdout | 2025-11-04 09:27:46,896 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:27:46,910 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:27:46,910 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:27:46,920 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:27:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:57.122849+00:00 (in 9.999578 seconds) proxycacheblobworker stdout | 2025-11-04 09:27:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:57 GMT)" (scheduled at 2025-11-04 09:27:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:27:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:27:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 47, 123589), True, datetime.datetime(2025, 11, 4, 9, 27, 47, 123589), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:27:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:27:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:27:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:27:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:27:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:15.479769+00:00 (in 28.000287 seconds) securityworker stdout | 2025-11-04 09:27:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:17 GMT)" (scheduled at 2025-11-04 09:27:47.479080+00:00) securityworker stdout | 2025-11-04 09:27:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:27:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:27:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:27:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:27:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:27:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 22, 47, 483043), 1, 2]) securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:27:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 22, 47, 483043), 1, 2]) securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:27:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:27:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:27:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:27:48,099 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:27:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:27:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:33.622985+00:00 (in 44.997318 seconds) exportactionlogsworker stdout | 2025-11-04 09:27:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:48 GMT)" (scheduled at 2025-11-04 09:27:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:27:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:27:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:27:48,765 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:27:50,070 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:27:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:27:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:21.226294+00:00 (in 29.999551 seconds) autopruneworker stdout | 2025-11-04 09:27:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:21 GMT)" (scheduled at 2025-11-04 09:27:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:27:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248411235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:27:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:27:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:27:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:27:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:27:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:01.530064+00:00 (in 9.999535 seconds) notificationworker stdout | 2025-11-04 09:27:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:01 GMT)" (scheduled at 2025-11-04 09:27:51.530064+00:00) notificationworker stdout | 2025-11-04 09:27:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:27:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 51, 530822), True, datetime.datetime(2025, 11, 4, 9, 27, 51, 530822), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:27:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:27:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:27:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:27:52,353 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:27:55,200 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:27:56,920 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:27:56,934 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:27:56,934 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:27:56,944 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:27:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:07.122849+00:00 (in 9.999623 seconds) proxycacheblobworker stdout | 2025-11-04 09:27:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:07 GMT)" (scheduled at 2025-11-04 09:27:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:27:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:27:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 27, 57, 123470), True, datetime.datetime(2025, 11, 4, 9, 27, 57, 123470), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:27:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:27:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:09.133645+00:00 (in 12.001459 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:27:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:57 GMT)" (scheduled at 2025-11-04 09:27:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:27:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:27:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:27:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:27:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:27:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:27:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:27:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:27:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:27:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:27.222846+00:00 (in 29.999536 seconds) buildlogsarchiver stdout | 2025-11-04 09:27:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:27 GMT)" (scheduled at 2025-11-04 09:27:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:27:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 27, 57, 223570), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:27:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:27:57,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:27:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:27:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:27:57,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:27:57,434 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:27:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:27:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:43.432867+00:00 (in 45.908573 seconds) repositorygcworker stdout | 2025-11-04 09:27:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:57 GMT)" (scheduled at 2025-11-04 09:27:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:27:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:27:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:28:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:27:58,142 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:27:58,797 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:27:59,179 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:27:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:27:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:14.351216+00:00 (in 14.999625 seconds) quotatotalworker stdout | 2025-11-04 09:27:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:14 GMT)" (scheduled at 2025-11-04 09:27:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:27:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:27:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:27:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:27:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:27:59,559 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:59,799 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:27:59,995 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:00,012 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:00,091 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:00,697 [288] [DEBUG] [app] Starting request: urn:request:98db4998-352b-4fe8-acf8-4787485d9870 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:00,701 [299] [DEBUG] [app] Starting request: urn:request:68661648-982e-4713-981f-8bce959108ed (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:00,701 [299] [DEBUG] [app] Ending request: urn:request:68661648-982e-4713-981f-8bce959108ed (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:68661648-982e-4713-981f-8bce959108ed', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:28:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:00,705 [285] [DEBUG] [app] Starting request: urn:request:30be3c51-800a-4875-8894-b5a4956e9773 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:00,706 [285] [DEBUG] [app] Ending request: urn:request:30be3c51-800a-4875-8894-b5a4956e9773 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:30be3c51-800a-4875-8894-b5a4956e9773', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:28:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:00,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:00,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:00,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:00,730 [288] [DEBUG] [app] Ending request: urn:request:98db4998-352b-4fe8-acf8-4787485d9870 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:98db4998-352b-4fe8-acf8-4787485d9870', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:00,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:28:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:28:00,784 [287] [DEBUG] [app] Starting request: urn:request:51c58827-46f0-4701-bc39-f27574513f51 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:00,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:00,788 [298] [DEBUG] [app] Starting request: urn:request:9f20bb72-29f0-4677-bfea-27fe2385b900 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:00,788 [298] [DEBUG] [app] Ending request: urn:request:9f20bb72-29f0-4677-bfea-27fe2385b900 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9f20bb72-29f0-4677-bfea-27fe2385b900', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:28:00,789 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:00,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:00,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:00,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:00,792 [285] [DEBUG] [app] Starting request: urn:request:79a62248-2b84-49a4-a6af-5fa7df78ce03 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:00,792 [285] [DEBUG] [app] Ending request: urn:request:79a62248-2b84-49a4-a6af-5fa7df78ce03 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:79a62248-2b84-49a4-a6af-5fa7df78ce03', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:28:00,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:00,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:00,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:00,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:00,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:00,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:00,811 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:00,814 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:00,817 [287] [DEBUG] [app] Ending request: urn:request:51c58827-46f0-4701-bc39-f27574513f51 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:51c58827-46f0-4701-bc39-f27574513f51', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:00,817 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:28:00,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:28:01,436 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:28:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:28:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:11.530064+00:00 (in 9.999555 seconds) notificationworker stdout | 2025-11-04 09:28:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:11 GMT)" (scheduled at 2025-11-04 09:28:01.530064+00:00) notificationworker stdout | 2025-11-04 09:28:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:28:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 1, 530773), True, datetime.datetime(2025, 11, 4, 9, 28, 1, 530773), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:28:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:28:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:28:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:28:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:28:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:39.924652+00:00 (in 38.001411 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:28:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:01 GMT)" (scheduled at 2025-11-04 09:28:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:28:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:28:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 1, 923542), True, datetime.datetime(2025, 11, 4, 9, 28, 1, 923542), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:28:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:28:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:28:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:28:02,222 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:28:02,420 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:28:02,422 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:28:03,709 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:28:03,710 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:28:03,815 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:28:04,587 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:28:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:28:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:08.437104+00:00 (in 3.000884 seconds) gcworker stdout | 2025-11-04 09:28:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:35 GMT)" (scheduled at 2025-11-04 09:28:05.435809+00:00) gcworker stdout | 2025-11-04 09:28:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038885436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:28:05,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:28:05,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:28:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:28:05,897 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:28:06,269 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:28:06,885 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:28:06,945 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:28:06,958 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:28:06,958 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:28:06,968 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:28:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:17.122849+00:00 (in 9.999585 seconds) proxycacheblobworker stdout | 2025-11-04 09:28:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:17 GMT)" (scheduled at 2025-11-04 09:28:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:28:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:28:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 7, 123557), True, datetime.datetime(2025, 11, 4, 9, 28, 7, 123557), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:28:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:28:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:28:07,305 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,291 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,291 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,305 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,324 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,345 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,359 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,387 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:28:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:28:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:21.434460+00:00 (in 12.998645 seconds) namespacegcworker stdout | 2025-11-04 09:28:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:33:08 GMT)" (scheduled at 2025-11-04 09:28:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:28:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 28, 8, 436068), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:28:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:28:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:35.435809+00:00 (in 26.998346 seconds) gcworker stdout | 2025-11-04 09:28:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:38 GMT)" (scheduled at 2025-11-04 09:28:08.437104+00:00) gcworker stdout | 2025-11-04 09:28:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) namespacegcworker stdout | 2025-11-04 09:28:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 28, 8, 436068), True, datetime.datetime(2025, 11, 4, 9, 28, 8, 436068), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:28:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248428449, None, 1, 0]) namespacegcworker stdout | 2025-11-04 09:28:08,452 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 28, 8, 436068), True, datetime.datetime(2025, 11, 4, 9, 28, 8, 436068), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 28, 8, 436068), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:28:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:28:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:28:08,455 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:28:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:33:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:28:08,541 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:08,542 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:28:08,714 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:28:08,715 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:28:08,719 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:28:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:28:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:57.131795+00:00 (in 47.997704 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:28:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:09 GMT)" (scheduled at 2025-11-04 09:28:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:28:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:28:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:28:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:28:09,147 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:28:09,437 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:28:11,215 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:28:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:28:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:15.535095+00:00 (in 4.004582 seconds) notificationworker stdout | 2025-11-04 09:28:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:21 GMT)" (scheduled at 2025-11-04 09:28:11.530064+00:00) notificationworker stdout | 2025-11-04 09:28:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:28:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 11, 530787), True, datetime.datetime(2025, 11, 4, 9, 28, 11, 530787), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:28:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:28:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:28:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:28:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:28:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:12.222828+00:00 (in 59.999592 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:28:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:12 GMT)" (scheduled at 2025-11-04 09:28:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:28:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:28:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:28:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:28:12,660 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:28:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:28:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:29.351216+00:00 (in 14.999578 seconds) quotatotalworker stdout | 2025-11-04 09:28:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:29 GMT)" (scheduled at 2025-11-04 09:28:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:28:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:28:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:28:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:28:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:28:15,164 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:28:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:28:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:17.479080+00:00 (in 1.998859 seconds) securityworker stdout | 2025-11-04 09:28:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:45 GMT)" (scheduled at 2025-11-04 09:28:15.479769+00:00) securityworker stdout | 2025-11-04 09:28:15,481 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:28:15,481 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:28:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:28:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:15,496 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:28:15,496 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:15,496 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:28:15,500 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:15,501 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 23, 15, 484433), 1, 2]) securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:28:15,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:15,505 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 23, 15, 484433), 1, 2]) securityworker stdout | 2025-11-04 09:28:15,508 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:15,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:15,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:15,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:15,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:15,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:15,509 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:15,509 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:15,509 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:28:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:28:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:21.530064+00:00 (in 5.994542 seconds) notificationworker stdout | 2025-11-04 09:28:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:15 GMT)" (scheduled at 2025-11-04 09:28:15.535095+00:00) notificationworker stdout | 2025-11-04 09:28:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:28:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:28:15,697 [287] [DEBUG] [app] Starting request: urn:request:350d393b-3090-436c-90d1-b1ad4e47f15e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:15,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:15,701 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:15,701 [292] [DEBUG] [app] Starting request: urn:request:9beb4c82-87d3-41c8-aede-b76bf8cdaa08 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:15,702 [292] [DEBUG] [app] Ending request: urn:request:9beb4c82-87d3-41c8-aede-b76bf8cdaa08 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9beb4c82-87d3-41c8-aede-b76bf8cdaa08', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:28:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:15,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:15,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:15,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:15,706 [285] [DEBUG] [app] Starting request: urn:request:50e52b2a-8a68-420e-9b4f-ff5c584f2ee5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:15,706 [285] [DEBUG] [app] Ending request: urn:request:50e52b2a-8a68-420e-9b4f-ff5c584f2ee5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:50e52b2a-8a68-420e-9b4f-ff5c584f2ee5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:15,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:15,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:15,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:15,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:15,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:15,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:15,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:15,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:15,730 [287] [DEBUG] [app] Ending request: urn:request:350d393b-3090-436c-90d1-b1ad4e47f15e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:350d393b-3090-436c-90d1-b1ad4e47f15e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:15,730 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:28:15,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:28:15,784 [285] [DEBUG] [app] Starting request: urn:request:d4c7da12-38b1-4115-94d4-493f884c3dc2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:15,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:15,788 [299] [DEBUG] [app] Starting request: urn:request:6ce75a9f-09cf-4e17-950a-e0a552e007a1 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:15,789 [299] [DEBUG] [app] Ending request: urn:request:6ce75a9f-09cf-4e17-950a-e0a552e007a1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6ce75a9f-09cf-4e17-950a-e0a552e007a1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:28:15,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:15,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:15,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:15,792 [287] [DEBUG] [app] Starting request: urn:request:f173ab65-5112-4f12-ac6e-d4eece582630 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:15,793 [287] [DEBUG] [app] Ending request: urn:request:f173ab65-5112-4f12-ac6e-d4eece582630 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f173ab65-5112-4f12-ac6e-d4eece582630', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:28:15,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:15,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:15,817 [285] [DEBUG] [app] Ending request: urn:request:d4c7da12-38b1-4115-94d4-493f884c3dc2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d4c7da12-38b1-4115-94d4-493f884c3dc2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:15,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:28:15,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) builder stdout | 2025-11-04 09:28:16,968 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:28:16,982 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:28:16,982 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:28:16,992 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:28:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:27.122849+00:00 (in 9.999615 seconds) proxycacheblobworker stdout | 2025-11-04 09:28:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:27 GMT)" (scheduled at 2025-11-04 09:28:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:28:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:28:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 17, 123520), True, datetime.datetime(2025, 11, 4, 9, 28, 17, 123520), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:28:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:28:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:28:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:28:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:45.479769+00:00 (in 28.000238 seconds) securityworker stdout | 2025-11-04 09:28:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:47 GMT)" (scheduled at 2025-11-04 09:28:17.479080+00:00) securityworker stdout | 2025-11-04 09:28:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:28:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:28:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:28:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:28:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:28:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:28:18,108 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:28:18,774 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:28:20,079 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:28:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:28:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:51.226294+00:00 (in 29.999601 seconds) autopruneworker stdout | 2025-11-04 09:28:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:51 GMT)" (scheduled at 2025-11-04 09:28:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:28:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248441234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:28:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:28:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:28:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:28:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:28:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:32.435813+00:00 (in 11.000932 seconds) namespacegcworker stdout | 2025-11-04 09:28:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:21 GMT)" (scheduled at 2025-11-04 09:28:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:28:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:28:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 21, 435191), True, datetime.datetime(2025, 11, 4, 9, 28, 21, 435191), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:28:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:28:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:28:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:28:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:28:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:31.530064+00:00 (in 9.999602 seconds) notificationworker stdout | 2025-11-04 09:28:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:31 GMT)" (scheduled at 2025-11-04 09:28:21.530064+00:00) notificationworker stdout | 2025-11-04 09:28:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:28:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 21, 530734), True, datetime.datetime(2025, 11, 4, 9, 28, 21, 530734), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:28:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:28:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:28:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:28:22,362 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:28:25,209 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:28:26,992 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:28:27,005 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:28:27,005 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:28:27,015 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:28:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:37.122849+00:00 (in 9.999636 seconds) proxycacheblobworker stdout | 2025-11-04 09:28:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:37 GMT)" (scheduled at 2025-11-04 09:28:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:28:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:28:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 27, 123448), True, datetime.datetime(2025, 11, 4, 9, 28, 27, 123448), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:28:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:28:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:28:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:28:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:57.222846+00:00 (in 29.999587 seconds) buildlogsarchiver stdout | 2025-11-04 09:28:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:57 GMT)" (scheduled at 2025-11-04 09:28:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:28:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 28, 27, 223465), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:28:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:28:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:28:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:28:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:28:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:28:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:28:27,443 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:28:28,151 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:28:28,806 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:28:29,188 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:28:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:28:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:44.351216+00:00 (in 14.999544 seconds) quotatotalworker stdout | 2025-11-04 09:28:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:44 GMT)" (scheduled at 2025-11-04 09:28:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:28:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:28:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:28:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:28:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:28:29,582 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:29,816 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:30,009 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:30,025 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:30,105 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:30,698 [288] [DEBUG] [app] Starting request: urn:request:c4dc7480-6272-4969-8e94-c684db5c1294 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:30,699 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:30,702 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:30,702 [296] [DEBUG] [app] Starting request: urn:request:d8065594-7aee-4215-a21e-33b69e555381 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:30,703 [296] [DEBUG] [app] Ending request: urn:request:d8065594-7aee-4215-a21e-33b69e555381 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d8065594-7aee-4215-a21e-33b69e555381', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:28:30,703 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:28:30,703 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:30,704 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:30,706 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:30,707 [285] [DEBUG] [app] Starting request: urn:request:1e6c3db7-23df-4736-ab76-9cb09325c6ce (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:30,707 [285] [DEBUG] [app] Ending request: urn:request:1e6c3db7-23df-4736-ab76-9cb09325c6ce (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1e6c3db7-23df-4736-ab76-9cb09325c6ce', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:28:30,708 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:30,708 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:30,708 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:30,708 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:30,708 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:30,716 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:30,716 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:30,726 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:30,729 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:30,732 [288] [DEBUG] [app] Ending request: urn:request:c4dc7480-6272-4969-8e94-c684db5c1294 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c4dc7480-6272-4969-8e94-c684db5c1294', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:30,732 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:28:30,732 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:28:30,784 [285] [DEBUG] [app] Starting request: urn:request:2c12cdf4-ae95-4d13-84f7-8607ec18fdf3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:30,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:30,788 [299] [DEBUG] [app] Starting request: urn:request:09922dd0-3b28-4f80-a9b1-c46d83fa8500 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:30,788 [299] [DEBUG] [app] Ending request: urn:request:09922dd0-3b28-4f80-a9b1-c46d83fa8500 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:09922dd0-3b28-4f80-a9b1-c46d83fa8500', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:28:30,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:30,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:30,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:30,792 [288] [DEBUG] [app] Starting request: urn:request:ba7cf83b-d1aa-41aa-803f-96484e3cfa40 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:30,792 [288] [DEBUG] [app] Ending request: urn:request:ba7cf83b-d1aa-41aa-803f-96484e3cfa40 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ba7cf83b-d1aa-41aa-803f-96484e3cfa40', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:28:30,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:30,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:30,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:30,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:30,817 [285] [DEBUG] [app] Ending request: urn:request:2c12cdf4-ae95-4d13-84f7-8607ec18fdf3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2c12cdf4-ae95-4d13-84f7-8607ec18fdf3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:30,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:28:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:28:31,445 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:28:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:28:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:41.530064+00:00 (in 9.999600 seconds) notificationworker stdout | 2025-11-04 09:28:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:41 GMT)" (scheduled at 2025-11-04 09:28:31.530064+00:00) notificationworker stdout | 2025-11-04 09:28:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:28:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 31, 530752), True, datetime.datetime(2025, 11, 4, 9, 28, 31, 530752), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:28:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:28:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:28:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:28:32,241 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:28:32,429 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:28:32,431 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:28:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:28:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:21.434460+00:00 (in 48.998228 seconds) namespacegcworker stdout | 2025-11-04 09:28:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:32 GMT)" (scheduled at 2025-11-04 09:28:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:28:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:28:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:32 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:28:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:28:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:48.625192+00:00 (in 15.001803 seconds) exportactionlogsworker stdout | 2025-11-04 09:28:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:33 GMT)" (scheduled at 2025-11-04 09:28:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:28:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:28:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 33, 623688), True, datetime.datetime(2025, 11, 4, 9, 28, 33, 623688), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:28:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:28:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:28:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:28:33,717 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:28:33,719 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:28:33,823 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:28:34,597 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:28:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:28:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:38.437104+00:00 (in 3.000885 seconds) gcworker stdout | 2025-11-04 09:28:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:05 GMT)" (scheduled at 2025-11-04 09:28:35.435809+00:00) gcworker stdout | 2025-11-04 09:28:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038915436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:28:35,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:28:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:28:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:28:35,906 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:28:36,277 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:28:36,894 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:28:37,016 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:28:37,029 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:28:37,029 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:28:37,040 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:28:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:40.124445+00:00 (in 3.001183 seconds) proxycacheblobworker stdout | 2025-11-04 09:28:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:47 GMT)" (scheduled at 2025-11-04 09:28:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:28:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:28:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 37, 123562), True, datetime.datetime(2025, 11, 4, 9, 28, 37, 123562), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:28:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:28:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:28:37,319 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,301 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,306 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,318 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,347 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,355 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,371 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,399 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:28:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:28:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:05.435809+00:00 (in 26.998302 seconds) gcworker stdout | 2025-11-04 09:28:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:08 GMT)" (scheduled at 2025-11-04 09:28:38.437104+00:00) gcworker stdout | 2025-11-04 09:28:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:28:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248458448, None, 1, 0]) gcworker stdout | 2025-11-04 09:28:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:28:38,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:28:38,553 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:28:38,553 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:28:38,724 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:28:38,725 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:28:38,729 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:28:39,456 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:28:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:28:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:01.922827+00:00 (in 21.997735 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:28:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:39 GMT)" (scheduled at 2025-11-04 09:28:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:28:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:28:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:28:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:28:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:47.122849+00:00 (in 6.997967 seconds) proxycacheblobworker stdout | 2025-11-04 09:28:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:40 GMT)" (scheduled at 2025-11-04 09:28:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:28:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:28:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:28:41,224 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:28:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:28:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:51.530064+00:00 (in 9.999534 seconds) notificationworker stdout | 2025-11-04 09:28:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:51 GMT)" (scheduled at 2025-11-04 09:28:41.530064+00:00) notificationworker stdout | 2025-11-04 09:28:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:28:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 41, 530860), True, datetime.datetime(2025, 11, 4, 9, 28, 41, 530860), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:28:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:28:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:28:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:28:42,669 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:28:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:28:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:57.523915+00:00 (in 14.090664 seconds) repositorygcworker stdout | 2025-11-04 09:28:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:43 GMT)" (scheduled at 2025-11-04 09:28:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:28:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:28:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 43, 433482), True, datetime.datetime(2025, 11, 4, 9, 28, 43, 433482), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:28:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:28:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:28:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:28:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:28:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:59.351216+00:00 (in 14.999620 seconds) quotatotalworker stdout | 2025-11-04 09:28:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:59 GMT)" (scheduled at 2025-11-04 09:28:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:28:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:28:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:28:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:28:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:28:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:28:45,173 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:28:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:28:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:47.479080+00:00 (in 1.998944 seconds) securityworker stdout | 2025-11-04 09:28:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:15 GMT)" (scheduled at 2025-11-04 09:28:45.479769+00:00) securityworker stdout | 2025-11-04 09:28:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:28:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:28:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:28:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:28:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:28:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 23, 45, 483747), 1, 2]) securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 23, 45, 483747), 1, 2]) securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:28:45,697 [285] [DEBUG] [app] Starting request: urn:request:683ba860-74ba-4161-8edf-26c7c68355f9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:45,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:45,702 [292] [DEBUG] [app] Starting request: urn:request:0d8542d9-3edc-4019-8d66-9a943a4b9240 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:45,702 [292] [DEBUG] [app] Ending request: urn:request:0d8542d9-3edc-4019-8d66-9a943a4b9240 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0d8542d9-3edc-4019-8d66-9a943a4b9240', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:28:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:28:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:45,706 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:45,706 [288] [DEBUG] [app] Starting request: urn:request:9c460641-e7c4-42f3-8cc8-116805a2c343 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:45,707 [288] [DEBUG] [app] Ending request: urn:request:9c460641-e7c4-42f3-8cc8-116805a2c343 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9c460641-e7c4-42f3-8cc8-116805a2c343', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:28:45,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:45,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:45,708 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:45,708 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:45,708 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:45,716 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:45,716 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:45,726 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:45,729 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:45,732 [285] [DEBUG] [app] Ending request: urn:request:683ba860-74ba-4161-8edf-26c7c68355f9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:683ba860-74ba-4161-8edf-26c7c68355f9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:45,732 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:28:45,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.037) gunicorn-web stdout | 2025-11-04 09:28:45,783 [285] [DEBUG] [app] Starting request: urn:request:c5786035-17c4-40e4-973c-bbb3ba4568cc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:28:45,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:28:45,785 [292] [DEBUG] [app] Starting request: urn:request:d5bee75e-bad7-475d-bf8f-ec84ba2ccf57 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:28:45,785 [292] [DEBUG] [app] Ending request: urn:request:d5bee75e-bad7-475d-bf8f-ec84ba2ccf57 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d5bee75e-bad7-475d-bf8f-ec84ba2ccf57', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:28:45,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:45,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:28:45,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:28:45,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:28:45,789 [288] [DEBUG] [app] Starting request: urn:request:a748a799-f26d-474f-bf34-813b2f9f9c73 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:28:45,789 [288] [DEBUG] [app] Ending request: urn:request:a748a799-f26d-474f-bf34-813b2f9f9c73 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a748a799-f26d-474f-bf34-813b2f9f9c73', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:28:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:28:45,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:28:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:28:45,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:28:45,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:28:45,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:28:45,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:28:45,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:28:45,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:28:45,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:28:45,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:28:45,814 [285] [DEBUG] [app] Ending request: urn:request:c5786035-17c4-40e4-973c-bbb3ba4568cc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c5786035-17c4-40e4-973c-bbb3ba4568cc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:28:45,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:28:45,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:28:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:28:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.032) builder stdout | 2025-11-04 09:28:47,040 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:28:47,053 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:28:47,053 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:28:47,063 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:28:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:28:57.122849+00:00 (in 9.999636 seconds) proxycacheblobworker stdout | 2025-11-04 09:28:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:57 GMT)" (scheduled at 2025-11-04 09:28:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:28:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:28:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 47, 123440), True, datetime.datetime(2025, 11, 4, 9, 28, 47, 123440), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:28:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:28:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:28:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:28:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:28:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:15.479769+00:00 (in 28.000237 seconds) securityworker stdout | 2025-11-04 09:28:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:17 GMT)" (scheduled at 2025-11-04 09:28:47.479080+00:00) securityworker stdout | 2025-11-04 09:28:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:28:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:28:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:28:47,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:28:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:28:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 23, 47, 484119), 1, 2]) securityworker stdout | 2025-11-04 09:28:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:28:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:28:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:28:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:28:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 23, 47, 484119), 1, 2]) securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:28:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:28:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:28:48,117 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:28:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:28:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:33.622985+00:00 (in 44.997373 seconds) exportactionlogsworker stdout | 2025-11-04 09:28:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:48 GMT)" (scheduled at 2025-11-04 09:28:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:28:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:28:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:28:48,782 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:28:50,088 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:28:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:28:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:21.226294+00:00 (in 29.999562 seconds) autopruneworker stdout | 2025-11-04 09:28:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:21 GMT)" (scheduled at 2025-11-04 09:28:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:28:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248471235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:28:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:28:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:28:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:28:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:28:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:01.530064+00:00 (in 9.999546 seconds) notificationworker stdout | 2025-11-04 09:28:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:01 GMT)" (scheduled at 2025-11-04 09:28:51.530064+00:00) notificationworker stdout | 2025-11-04 09:28:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:28:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 51, 530831), True, datetime.datetime(2025, 11, 4, 9, 28, 51, 530831), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:28:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:28:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:28:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:28:52,370 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:28:55,218 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:28:57,063 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:28:57,076 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:28:57,076 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:28:57,086 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:28:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:07.122849+00:00 (in 9.999615 seconds) proxycacheblobworker stdout | 2025-11-04 09:28:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:07 GMT)" (scheduled at 2025-11-04 09:28:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:28:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:28:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 28, 57, 123532), True, datetime.datetime(2025, 11, 4, 9, 28, 57, 123532), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:28:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:28:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:09.133645+00:00 (in 12.001482 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:28:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:57 GMT)" (scheduled at 2025-11-04 09:28:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:28:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:28:57,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:28:57,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:28:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:28:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:28:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:28:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:28:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:28:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:27.222846+00:00 (in 29.999452 seconds) buildlogsarchiver stdout | 2025-11-04 09:28:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:27 GMT)" (scheduled at 2025-11-04 09:28:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:28:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 28, 57, 223636), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:28:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:28:57,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:28:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:28:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:28:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:28:57,451 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:28:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:28:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:02.523412+00:00 (in 4.999106 seconds) repositorygcworker stdout | 2025-11-04 09:28:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:57 GMT)" (scheduled at 2025-11-04 09:28:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:28:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:28:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:29:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:28:58,160 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:28:58,814 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:28:59,196 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:28:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:28:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:14.351216+00:00 (in 14.999559 seconds) quotatotalworker stdout | 2025-11-04 09:28:59,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:14 GMT)" (scheduled at 2025-11-04 09:28:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:28:59,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:28:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:28:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:28:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:28:59,593 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:28:59,828 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:00,023 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:00,038 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:00,120 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:00,696 [285] [DEBUG] [app] Starting request: urn:request:06988f9f-f53a-4262-a457-1b65396e015f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:00,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:00,701 [292] [DEBUG] [app] Starting request: urn:request:0519c8a5-aa79-490d-89d3-8d9d6980d815 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:00,701 [292] [DEBUG] [app] Ending request: urn:request:0519c8a5-aa79-490d-89d3-8d9d6980d815 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0519c8a5-aa79-490d-89d3-8d9d6980d815', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:29:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:29:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:00,706 [288] [DEBUG] [app] Starting request: urn:request:d6d996b4-6771-4ee8-aac3-ae0dadc2c24a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:00,706 [288] [DEBUG] [app] Ending request: urn:request:d6d996b4-6771-4ee8-aac3-ae0dadc2c24a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d6d996b4-6771-4ee8-aac3-ae0dadc2c24a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:29:00,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:29:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:00,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:00,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:00,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:00,731 [285] [DEBUG] [app] Ending request: urn:request:06988f9f-f53a-4262-a457-1b65396e015f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:06988f9f-f53a-4262-a457-1b65396e015f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:29:00,732 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.037 47 0.036) gunicorn-web stdout | 2025-11-04 09:29:00,784 [285] [DEBUG] [app] Starting request: urn:request:3dceb0fa-273d-4eb8-acac-73bbe656c96d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:00,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:00,785 [292] [DEBUG] [app] Starting request: urn:request:981c6f98-4b0d-49ad-b1fc-c69be00fa931 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:00,786 [292] [DEBUG] [app] Ending request: urn:request:981c6f98-4b0d-49ad-b1fc-c69be00fa931 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:981c6f98-4b0d-49ad-b1fc-c69be00fa931', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:29:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:29:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:00,790 [288] [DEBUG] [app] Starting request: urn:request:034f3543-2d64-4afd-bfec-917b4055daeb (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:00,790 [288] [DEBUG] [app] Ending request: urn:request:034f3543-2d64-4afd-bfec-917b4055daeb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:034f3543-2d64-4afd-bfec-917b4055daeb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:29:00,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:00,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:00,791 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:00,791 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:00,791 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:00,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:00,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:00,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:00,814 [285] [DEBUG] [app] Ending request: urn:request:3dceb0fa-273d-4eb8-acac-73bbe656c96d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3dceb0fa-273d-4eb8-acac-73bbe656c96d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:00,814 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) gunicorn-web stdout | 2025-11-04 09:29:00,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:29:01,454 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:29:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:11.530064+00:00 (in 9.999599 seconds) notificationworker stdout | 2025-11-04 09:29:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:11 GMT)" (scheduled at 2025-11-04 09:29:01.530064+00:00) notificationworker stdout | 2025-11-04 09:29:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:29:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 1, 530798), True, datetime.datetime(2025, 11, 4, 9, 29, 1, 530798), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:29:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:29:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:29:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:29:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:29:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:39.924652+00:00 (in 38.001451 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:29:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:01 GMT)" (scheduled at 2025-11-04 09:29:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:29:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:29:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 1, 923455), True, datetime.datetime(2025, 11, 4, 9, 29, 1, 923455), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:29:01,934 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:29:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:29:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:29:02,250 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:29:02,437 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:29:02,440 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:29:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:29:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:43.432867+00:00 (in 40.909071 seconds) repositorygcworker stdout | 2025-11-04 09:29:02,523 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:34:02 GMT)" (scheduled at 2025-11-04 09:29:02.523412+00:00) repositorygcworker stdout | 2025-11-04 09:29:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 29, 2, 524009), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:29:02,535 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 29, 2, 524009), True, datetime.datetime(2025, 11, 4, 9, 29, 2, 524009), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:29:02,539 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 29, 2, 524009), True, datetime.datetime(2025, 11, 4, 9, 29, 2, 524009), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 29, 2, 524009), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:29:02,542 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:29:02,542 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:34:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:29:03,726 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:29:03,728 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:29:03,832 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:29:04,606 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:29:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:29:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:08.437104+00:00 (in 3.000909 seconds) gcworker stdout | 2025-11-04 09:29:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:35 GMT)" (scheduled at 2025-11-04 09:29:05.435809+00:00) gcworker stdout | 2025-11-04 09:29:05,436 [72] [DEBUG] [peewee] ('SELECT DISTINCT "t1"."removed_tag_expiration_s" FROM "user" AS "t1" LIMIT %s', [100]) gcworker stdout | 2025-11-04 09:29:05,448 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038945448, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:29:05,453 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:29:05,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:29:05,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:29:05,915 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:29:06,286 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:29:06,904 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:29:07,086 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:29:07,099 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:29:07,099 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:29:07,109 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:17.122849+00:00 (in 9.999591 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:17 GMT)" (scheduled at 2025-11-04 09:29:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:29:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 7, 123546), True, datetime.datetime(2025, 11, 4, 9, 29, 7, 123546), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:29:07,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:29:07,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:07,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:29:07,328 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,311 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,320 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,331 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,359 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,371 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,383 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,411 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:29:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:29:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:35.435809+00:00 (in 26.998297 seconds) gcworker stdout | 2025-11-04 09:29:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:38 GMT)" (scheduled at 2025-11-04 09:29:08.437104+00:00) gcworker stdout | 2025-11-04 09:29:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:29:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248488449, None, 1, 0]) gcworker stdout | 2025-11-04 09:29:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:29:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:29:08,568 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:08,568 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:29:08,735 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:29:08,739 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:29:08,743 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:29:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:29:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:57.131795+00:00 (in 47.997737 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:29:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:09 GMT)" (scheduled at 2025-11-04 09:29:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:29:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:29:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:29:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:29:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:29:09,466 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:29:11,233 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:29:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:15.535095+00:00 (in 4.004540 seconds) notificationworker stdout | 2025-11-04 09:29:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:21 GMT)" (scheduled at 2025-11-04 09:29:11.530064+00:00) notificationworker stdout | 2025-11-04 09:29:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:29:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 11, 530908), True, datetime.datetime(2025, 11, 4, 9, 29, 11, 530908), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:29:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:29:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:29:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:29:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:29:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:12.222828+00:00 (in 59.999558 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:29:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:12 GMT)" (scheduled at 2025-11-04 09:29:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:29:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:29:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:29:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:29:12,678 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:29:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:29:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:29.351216+00:00 (in 14.999577 seconds) quotatotalworker stdout | 2025-11-04 09:29:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:29 GMT)" (scheduled at 2025-11-04 09:29:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:29:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:29:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:29:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:29:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:29:15,182 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:29:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:29:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:17.479080+00:00 (in 1.998894 seconds) securityworker stdout | 2025-11-04 09:29:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:45 GMT)" (scheduled at 2025-11-04 09:29:15.479769+00:00) securityworker stdout | 2025-11-04 09:29:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:29:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:29:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:29:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 24, 15, 483782), 1, 2]) securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 24, 15, 483782), 1, 2]) securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:29:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:21.530064+00:00 (in 5.994518 seconds) notificationworker stdout | 2025-11-04 09:29:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:15 GMT)" (scheduled at 2025-11-04 09:29:15.535095+00:00) notificationworker stdout | 2025-11-04 09:29:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:29:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:29:15,697 [285] [DEBUG] [app] Starting request: urn:request:61408e97-0df8-499f-a6ce-bbaa5b035425 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:15,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:15,701 [297] [DEBUG] [app] Starting request: urn:request:49400c19-2e93-4aca-8c3f-4c461a4fc569 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:15,702 [297] [DEBUG] [app] Ending request: urn:request:49400c19-2e93-4aca-8c3f-4c461a4fc569 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:49400c19-2e93-4aca-8c3f-4c461a4fc569', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:29:15,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:29:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:15,706 [288] [DEBUG] [app] Starting request: urn:request:465b0142-a604-4664-a8d1-613c36825b54 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:15,706 [288] [DEBUG] [app] Ending request: urn:request:465b0142-a604-4664-a8d1-613c36825b54 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:465b0142-a604-4664-a8d1-613c36825b54', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:29:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:15,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:15,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:15,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:15,731 [285] [DEBUG] [app] Ending request: urn:request:61408e97-0df8-499f-a6ce-bbaa5b035425 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:61408e97-0df8-499f-a6ce-bbaa5b035425', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:15,731 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:29:15,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:29:15,783 [288] [DEBUG] [app] Starting request: urn:request:0388fd36-e44c-4ad7-addb-7586bc2f324c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:15,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:15,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:15,788 [292] [DEBUG] [app] Starting request: urn:request:128127e3-07e0-4354-a48c-ebe9a86b6cf8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:15,788 [292] [DEBUG] [app] Ending request: urn:request:128127e3-07e0-4354-a48c-ebe9a86b6cf8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:128127e3-07e0-4354-a48c-ebe9a86b6cf8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:29:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:29:15,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:15,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:15,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:15,792 [285] [DEBUG] [app] Starting request: urn:request:634f0827-8db4-4f64-9753-5ac62c43be49 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:15,792 [285] [DEBUG] [app] Ending request: urn:request:634f0827-8db4-4f64-9753-5ac62c43be49 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:634f0827-8db4-4f64-9753-5ac62c43be49', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:15,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:29:15,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:15,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:15,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:15,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:15,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:15,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:15,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:15,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:15,817 [288] [DEBUG] [app] Ending request: urn:request:0388fd36-e44c-4ad7-addb-7586bc2f324c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0388fd36-e44c-4ad7-addb-7586bc2f324c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:15,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:29:15,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) builder stdout | 2025-11-04 09:29:17,110 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue proxycacheblobworker stdout | 2025-11-04 09:29:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:27.122849+00:00 (in 9.999647 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:27 GMT)" (scheduled at 2025-11-04 09:29:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. builder stdout | 2025-11-04 09:29:17,123 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:29:17,123 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics proxycacheblobworker stdout | 2025-11-04 09:29:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 17, 123475), True, datetime.datetime(2025, 11, 4, 9, 29, 17, 123475), 0, 'proxycacheblob/%', 50, 1, 0]) builder stdout | 2025-11-04 09:29:17,133 [66] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:29:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:29:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:29:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:45.479769+00:00 (in 28.000265 seconds) securityworker stdout | 2025-11-04 09:29:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:47 GMT)" (scheduled at 2025-11-04 09:29:17.479080+00:00) securityworker stdout | 2025-11-04 09:29:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:29:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:29:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:29:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:29:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:29:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:29:18,125 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:29:18,791 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:29:20,097 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:29:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:29:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:51.226294+00:00 (in 29.999536 seconds) autopruneworker stdout | 2025-11-04 09:29:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:51 GMT)" (scheduled at 2025-11-04 09:29:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:29:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248501234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:29:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:29:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:29:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:29:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:29:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:32.435813+00:00 (in 11.000939 seconds) namespacegcworker stdout | 2025-11-04 09:29:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:21 GMT)" (scheduled at 2025-11-04 09:29:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:29:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:29:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 21, 435151), True, datetime.datetime(2025, 11, 4, 9, 29, 21, 435151), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:29:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:29:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:29:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:29:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:31.530064+00:00 (in 9.999587 seconds) notificationworker stdout | 2025-11-04 09:29:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:31 GMT)" (scheduled at 2025-11-04 09:29:21.530064+00:00) notificationworker stdout | 2025-11-04 09:29:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:29:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 21, 530775), True, datetime.datetime(2025, 11, 4, 9, 29, 21, 530775), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:29:21,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:29:21,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:29:21,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:29:22,379 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:29:25,227 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:29:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:37.122849+00:00 (in 9.999638 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:37 GMT)" (scheduled at 2025-11-04 09:29:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:29:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 27, 123447), True, datetime.datetime(2025, 11, 4, 9, 29, 27, 123447), 0, 'proxycacheblob/%', 50, 1, 0]) builder stdout | 2025-11-04 09:29:27,134 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue proxycacheblobworker stdout | 2025-11-04 09:29:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:29:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:37 GMT)" executed successfully builder stdout | 2025-11-04 09:29:27,146 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:29:27,147 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:29:27,157 [66] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:29:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:29:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:57.222846+00:00 (in 29.999544 seconds) buildlogsarchiver stdout | 2025-11-04 09:29:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:57 GMT)" (scheduled at 2025-11-04 09:29:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:29:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 29, 27, 223568), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:29:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:29:27,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:29:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:29:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:29:27,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:29:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:29:27,460 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:29:28,169 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:29:28,823 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:29:29,205 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:29:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:29:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:44.351216+00:00 (in 14.999606 seconds) quotatotalworker stdout | 2025-11-04 09:29:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:44 GMT)" (scheduled at 2025-11-04 09:29:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:29:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:29:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:29:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:29:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:29:29,603 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:29,847 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:30,036 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:30,051 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:30,133 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:30,696 [285] [DEBUG] [app] Starting request: urn:request:f455d9c2-ae01-46a8-a9b3-1acdb295c51b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:30,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:30,701 [292] [DEBUG] [app] Starting request: urn:request:689943ea-4a87-4378-8fea-97e5550785a7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:30,701 [292] [DEBUG] [app] Ending request: urn:request:689943ea-4a87-4378-8fea-97e5550785a7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:689943ea-4a87-4378-8fea-97e5550785a7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:29:30,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:29:30,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:30,705 [287] [DEBUG] [app] Starting request: urn:request:5ca91510-5d4c-465a-b1e6-ee904f866d98 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:30,705 [287] [DEBUG] [app] Ending request: urn:request:5ca91510-5d4c-465a-b1e6-ee904f866d98 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5ca91510-5d4c-465a-b1e6-ee904f866d98', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:29:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:30,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:30,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:30,730 [285] [DEBUG] [app] Ending request: urn:request:f455d9c2-ae01-46a8-a9b3-1acdb295c51b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f455d9c2-ae01-46a8-a9b3-1acdb295c51b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:30,730 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:29:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:29:30,783 [285] [DEBUG] [app] Starting request: urn:request:1faec090-389c-40ab-9703-ca06da6b4728 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:30,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:30,785 [292] [DEBUG] [app] Starting request: urn:request:fbe62f99-c974-48e8-8e01-2bb2b3fb494c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:30,785 [292] [DEBUG] [app] Ending request: urn:request:fbe62f99-c974-48e8-8e01-2bb2b3fb494c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fbe62f99-c974-48e8-8e01-2bb2b3fb494c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:29:30,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:29:30,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:30,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:30,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:30,789 [287] [DEBUG] [app] Starting request: urn:request:dba79444-2988-4e6c-a832-02ae09e2cd2d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:30,789 [287] [DEBUG] [app] Ending request: urn:request:dba79444-2988-4e6c-a832-02ae09e2cd2d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dba79444-2988-4e6c-a832-02ae09e2cd2d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:30,789 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:29:30,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:30,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:30,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:30,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:30,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:30,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:30,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:30,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:30,813 [285] [DEBUG] [app] Ending request: urn:request:1faec090-389c-40ab-9703-ca06da6b4728 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1faec090-389c-40ab-9703-ca06da6b4728', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:30,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:29:30,813 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.030 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:29:31,463 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:29:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:40.533243+00:00 (in 9.002700 seconds) notificationworker stdout | 2025-11-04 09:29:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:41 GMT)" (scheduled at 2025-11-04 09:29:31.530064+00:00) notificationworker stdout | 2025-11-04 09:29:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:29:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 31, 530838), True, datetime.datetime(2025, 11, 4, 9, 29, 31, 530838), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:29:31,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:29:31,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:29:31,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:29:32,258 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:29:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:29:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:21.434460+00:00 (in 48.998268 seconds) namespacegcworker stdout | 2025-11-04 09:29:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:32 GMT)" (scheduled at 2025-11-04 09:29:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:29:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:29:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:32 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:29:32,446 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:29:32,449 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:29:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:29:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:48.625192+00:00 (in 15.001828 seconds) exportactionlogsworker stdout | 2025-11-04 09:29:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:33 GMT)" (scheduled at 2025-11-04 09:29:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:29:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:29:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 33, 623616), True, datetime.datetime(2025, 11, 4, 9, 29, 33, 623616), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:29:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:29:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:29:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:29:33,735 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:29:33,736 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:29:33,841 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:29:34,615 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:29:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:29:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:38.437104+00:00 (in 3.000920 seconds) gcworker stdout | 2025-11-04 09:29:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:05 GMT)" (scheduled at 2025-11-04 09:29:35.435809+00:00) gcworker stdout | 2025-11-04 09:29:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038975436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:29:35,449 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:29:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:29:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:29:35,936 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:29:36,305 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:29:36,914 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:29:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:40.124445+00:00 (in 3.001226 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:47 GMT)" (scheduled at 2025-11-04 09:29:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:29:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 37, 123479), True, datetime.datetime(2025, 11, 4, 9, 29, 37, 123479), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:29:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:29:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:47 GMT)" executed successfully builder stdout | 2025-11-04 09:29:37,157 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:29:37,170 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:29:37,170 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:29:37,180 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:29:37,337 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,326 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,333 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,343 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,369 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,393 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,384 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,422 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:29:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:29:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:05.435809+00:00 (in 26.998330 seconds) gcworker stdout | 2025-11-04 09:29:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:08 GMT)" (scheduled at 2025-11-04 09:29:38.437104+00:00) gcworker stdout | 2025-11-04 09:29:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:29:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248518449, None, 1, 0]) gcworker stdout | 2025-11-04 09:29:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:29:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:29:38,579 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:29:38,583 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:29:38,746 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:29:38,749 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:29:38,754 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:29:39,475 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:29:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:29:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:01.922827+00:00 (in 21.997761 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:29:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:39 GMT)" (scheduled at 2025-11-04 09:29:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:29:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:29:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:29:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:47.122849+00:00 (in 6.997982 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:40 GMT)" (scheduled at 2025-11-04 09:29:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:29:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:29:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:41.530064+00:00 (in 0.996374 seconds) notificationworker stdout | 2025-11-04 09:29:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:34:40 GMT)" (scheduled at 2025-11-04 09:29:40.533243+00:00) notificationworker stdout | 2025-11-04 09:29:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 29, 40, 533949), 'notification/%']) notificationworker stdout | 2025-11-04 09:29:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 29, 40, 533949), True, datetime.datetime(2025, 11, 4, 9, 29, 40, 533949), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:29:40,549 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 29, 40, 533949), True, datetime.datetime(2025, 11, 4, 9, 29, 40, 533949), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 29, 40, 533949), 'notification/%']) notificationworker stdout | 2025-11-04 09:29:40,552 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:29:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:34:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:29:41,242 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:29:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:51.530064+00:00 (in 9.999551 seconds) notificationworker stdout | 2025-11-04 09:29:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:51 GMT)" (scheduled at 2025-11-04 09:29:41.530064+00:00) notificationworker stdout | 2025-11-04 09:29:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:29:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 41, 530803), True, datetime.datetime(2025, 11, 4, 9, 29, 41, 530803), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:29:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:29:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:29:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:29:42,687 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:29:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:29:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:57.523915+00:00 (in 14.090668 seconds) repositorygcworker stdout | 2025-11-04 09:29:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:43 GMT)" (scheduled at 2025-11-04 09:29:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:29:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:29:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 43, 433476), True, datetime.datetime(2025, 11, 4, 9, 29, 43, 433476), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:29:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:29:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:29:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:29:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:29:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:59.351216+00:00 (in 14.999618 seconds) quotatotalworker stdout | 2025-11-04 09:29:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:59 GMT)" (scheduled at 2025-11-04 09:29:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:29:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:29:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:29:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:29:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:29:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:29:45,190 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:29:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:29:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:47.479080+00:00 (in 1.998897 seconds) securityworker stdout | 2025-11-04 09:29:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:15 GMT)" (scheduled at 2025-11-04 09:29:45.479769+00:00) securityworker stdout | 2025-11-04 09:29:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:29:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:29:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:29:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:45,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:45,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:45,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:45,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:45,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 24, 45, 483835), 1, 2]) securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:45,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 24, 45, 483835), 1, 2]) securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:29:45,697 [288] [DEBUG] [app] Starting request: urn:request:7ae5e460-d12e-4249-9d17-798569c42e30 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:45,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:45,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:45,702 [298] [DEBUG] [app] Starting request: urn:request:b492676b-d936-4918-b1f7-a6fc39576989 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:45,702 [298] [DEBUG] [app] Ending request: urn:request:b492676b-d936-4918-b1f7-a6fc39576989 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b492676b-d936-4918-b1f7-a6fc39576989', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:29:45,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:29:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:45,706 [287] [DEBUG] [app] Starting request: urn:request:9af2191a-1689-4cf9-8f25-c792643cec27 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:45,706 [287] [DEBUG] [app] Ending request: urn:request:9af2191a-1689-4cf9-8f25-c792643cec27 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9af2191a-1689-4cf9-8f25-c792643cec27', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:29:45,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:45,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:45,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:45,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:45,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:45,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:45,731 [288] [DEBUG] [app] Ending request: urn:request:7ae5e460-d12e-4249-9d17-798569c42e30 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7ae5e460-d12e-4249-9d17-798569c42e30', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:45,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:29:45,732 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:29:45,784 [288] [DEBUG] [app] Starting request: urn:request:5c308ab8-bfc9-4a6f-a51d-a9c3c0590731 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:29:45,785 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:29:45,785 [292] [DEBUG] [app] Starting request: urn:request:ae4368e3-65c7-425a-afe1-644cd832dd6f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:29:45,786 [292] [DEBUG] [app] Ending request: urn:request:ae4368e3-65c7-425a-afe1-644cd832dd6f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ae4368e3-65c7-425a-afe1-644cd832dd6f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:29:45,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:29:45,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:45,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:29:45,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:29:45,790 [287] [DEBUG] [app] Starting request: urn:request:52bc93f2-9f49-436b-97a3-29a2b1f6bf1c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:29:45,790 [287] [DEBUG] [app] Ending request: urn:request:52bc93f2-9f49-436b-97a3-29a2b1f6bf1c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:52bc93f2-9f49-436b-97a3-29a2b1f6bf1c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:29:45,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:29:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:29:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:29:45,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:29:45,791 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:29:45,791 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:29:45,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:29:45,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:29:45,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:29:45,807 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:29:45,810 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:29:45,813 [288] [DEBUG] [app] Ending request: urn:request:5c308ab8-bfc9-4a6f-a51d-a9c3c0590731 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5c308ab8-bfc9-4a6f-a51d-a9c3c0590731', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:29:45,814 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:29:45,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:29:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:29:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.030) proxycacheblobworker stdout | 2025-11-04 09:29:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:53.124163+00:00 (in 6.000901 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:57 GMT)" (scheduled at 2025-11-04 09:29:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:29:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 47, 123585), True, datetime.datetime(2025, 11, 4, 9, 29, 47, 123585), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:29:47,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:29:47,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:47,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:29:57 GMT)" executed successfully builder stdout | 2025-11-04 09:29:47,180 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:29:47,193 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:29:47,193 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:29:47,203 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:29:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:29:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:15.479769+00:00 (in 28.000214 seconds) securityworker stdout | 2025-11-04 09:29:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:17 GMT)" (scheduled at 2025-11-04 09:29:47.479080+00:00) securityworker stdout | 2025-11-04 09:29:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:29:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:29:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:29:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:29:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:29:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 24, 47, 483160), 1, 2]) securityworker stdout | 2025-11-04 09:29:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:29:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 24, 47, 483160), 1, 2]) securityworker stdout | 2025-11-04 09:29:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:29:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:29:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:29:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:29:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:29:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:29:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:29:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:29:48,135 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:29:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:29:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:33.622985+00:00 (in 44.997411 seconds) exportactionlogsworker stdout | 2025-11-04 09:29:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:48 GMT)" (scheduled at 2025-11-04 09:29:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:29:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:29:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:29:48,799 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:29:50,106 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:29:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:29:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:21.226294+00:00 (in 29.999555 seconds) autopruneworker stdout | 2025-11-04 09:29:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:21 GMT)" (scheduled at 2025-11-04 09:29:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:29:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248531234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:29:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:29:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:29:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:29:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:29:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:01.530064+00:00 (in 9.999631 seconds) notificationworker stdout | 2025-11-04 09:29:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:01 GMT)" (scheduled at 2025-11-04 09:29:51.530064+00:00) notificationworker stdout | 2025-11-04 09:29:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:29:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 51, 530731), True, datetime.datetime(2025, 11, 4, 9, 29, 51, 530731), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:29:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:29:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:29:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:29:52,388 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:29:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:29:57.122849+00:00 (in 3.998257 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:34:53 GMT)" (scheduled at 2025-11-04 09:29:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 29, 53, 124859), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:29:53,137 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 29, 53, 124859), True, datetime.datetime(2025, 11, 4, 9, 29, 53, 124859), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:29:53,141 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 29, 53, 124859), True, datetime.datetime(2025, 11, 4, 9, 29, 53, 124859), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 29, 53, 124859), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:29:53,144 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:53,144 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:34:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:29:55,236 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:29:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:29:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:07.122849+00:00 (in 9.999612 seconds) proxycacheblobworker stdout | 2025-11-04 09:29:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:07 GMT)" (scheduled at 2025-11-04 09:29:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:29:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:29:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 29, 57, 123564), True, datetime.datetime(2025, 11, 4, 9, 29, 57, 123564), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:29:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:29:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:09.133645+00:00 (in 12.001456 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:29:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:57 GMT)" (scheduled at 2025-11-04 09:29:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:29:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:29:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:29:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:29:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:29:57,144 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:29:57,144 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:29:57,144 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:57 GMT)" executed successfully builder stdout | 2025-11-04 09:29:57,203 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:29:57,216 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:29:57,216 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics buildlogsarchiver stdout | 2025-11-04 09:29:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:29:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:27.222846+00:00 (in 29.999620 seconds) buildlogsarchiver stdout | 2025-11-04 09:29:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:27 GMT)" (scheduled at 2025-11-04 09:29:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:29:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 29, 57, 223407), False, 50, 1, 0]) builder stdout | 2025-11-04 09:29:57,227 [66] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:29:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:29:57,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:29:57,238 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:29:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:29:57,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:29:57,469 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:29:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:29:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:43.432867+00:00 (in 45.908571 seconds) repositorygcworker stdout | 2025-11-04 09:29:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:57 GMT)" (scheduled at 2025-11-04 09:29:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:29:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:29:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:30:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:29:58,178 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:29:58,832 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:29:59,213 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:29:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:29:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:14.351216+00:00 (in 14.999564 seconds) quotatotalworker stdout | 2025-11-04 09:29:59,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:14 GMT)" (scheduled at 2025-11-04 09:29:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:29:59,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:29:59,353 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:29:59,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:29:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:29:59,614 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:29:59,860 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:00,048 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:00,064 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:00,147 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:00,697 [285] [DEBUG] [app] Starting request: urn:request:89b705f3-a44b-429b-81de-da12f2062072 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:00,702 [298] [DEBUG] [app] Starting request: urn:request:ca6b2528-a1bc-4d82-94b2-37b3da99c804 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:00,702 [298] [DEBUG] [app] Ending request: urn:request:ca6b2528-a1bc-4d82-94b2-37b3da99c804 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ca6b2528-a1bc-4d82-94b2-37b3da99c804', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:30:00,703 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:30:00,703 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:00,704 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:00,706 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:00,707 [288] [DEBUG] [app] Starting request: urn:request:350026cd-6ba9-4b68-bc78-0d36066c1986 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:00,708 [288] [DEBUG] [app] Ending request: urn:request:350026cd-6ba9-4b68-bc78-0d36066c1986 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:350026cd-6ba9-4b68-bc78-0d36066c1986', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:30:00,708 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:30:00,708 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:30:00,709 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:00,709 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:00,709 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:00,717 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:00,717 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:00,727 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:00,731 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:00,734 [285] [DEBUG] [app] Ending request: urn:request:89b705f3-a44b-429b-81de-da12f2062072 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:89b705f3-a44b-429b-81de-da12f2062072', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:00,734 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.038 47 0.038) gunicorn-web stdout | 2025-11-04 09:30:00,735 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:30:00,784 [285] [DEBUG] [app] Starting request: urn:request:b272bbfd-6c52-460c-893b-e24d2486a6e0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:00,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:00,786 [292] [DEBUG] [app] Starting request: urn:request:e2567065-68a5-45fa-96c0-7946aac7d968 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:00,786 [292] [DEBUG] [app] Ending request: urn:request:e2567065-68a5-45fa-96c0-7946aac7d968 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e2567065-68a5-45fa-96c0-7946aac7d968', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:30:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:30:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:00,788 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:00,790 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:00,791 [288] [DEBUG] [app] Starting request: urn:request:c54c6cc9-799a-428a-9363-d8aa75aacd03 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:00,791 [288] [DEBUG] [app] Ending request: urn:request:c54c6cc9-799a-428a-9363-d8aa75aacd03 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c54c6cc9-799a-428a-9363-d8aa75aacd03', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:30:00,791 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:30:00,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:00,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:00,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:00,792 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:00,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:00,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:00,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:00,816 [285] [DEBUG] [app] Ending request: urn:request:b272bbfd-6c52-460c-893b-e24d2486a6e0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b272bbfd-6c52-460c-893b-e24d2486a6e0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:00,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:30:00,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) manifestsubjectbackfillworker stdout | 2025-11-04 09:30:01,472 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:30:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:30:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:11.530064+00:00 (in 9.999499 seconds) notificationworker stdout | 2025-11-04 09:30:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:11 GMT)" (scheduled at 2025-11-04 09:30:01.530064+00:00) notificationworker stdout | 2025-11-04 09:30:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:30:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 1, 530876), True, datetime.datetime(2025, 11, 4, 9, 30, 1, 530876), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:30:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:30:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:30:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:30:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:30:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:39.924652+00:00 (in 38.001407 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:30:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:01 GMT)" (scheduled at 2025-11-04 09:30:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:30:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:30:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 1, 923515), True, datetime.datetime(2025, 11, 4, 9, 30, 1, 923515), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:30:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:30:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:30:01,937 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:30:02,267 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:30:02,454 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:30:02,457 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:30:03,744 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:30:03,745 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:30:03,849 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:30:04,625 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:30:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:30:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:08.437104+00:00 (in 3.000901 seconds) gcworker stdout | 2025-11-04 09:30:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:35 GMT)" (scheduled at 2025-11-04 09:30:05.435809+00:00) gcworker stdout | 2025-11-04 09:30:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039005436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:30:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:30:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:30:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:30:05,944 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:30:06,314 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:30:06,923 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:30:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:30:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:17.122849+00:00 (in 9.999609 seconds) proxycacheblobworker stdout | 2025-11-04 09:30:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:17 GMT)" (scheduled at 2025-11-04 09:30:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:30:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:30:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 7, 123814), True, datetime.datetime(2025, 11, 4, 9, 30, 7, 123814), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:30:07,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:30:07,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:30:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:17 GMT)" executed successfully builder stdout | 2025-11-04 09:30:07,227 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:30:07,240 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:30:07,240 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:30:07,250 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:30:07,346 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,345 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,340 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,366 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,383 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,408 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,411 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,432 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:30:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:30:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:35.435809+00:00 (in 26.998351 seconds) gcworker stdout | 2025-11-04 09:30:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:38 GMT)" (scheduled at 2025-11-04 09:30:08.437104+00:00) gcworker stdout | 2025-11-04 09:30:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:30:08,450 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248548450, None, 1, 0]) gcworker stdout | 2025-11-04 09:30:08,454 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:30:08,454 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:30:08,592 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:08,598 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:30:08,756 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:30:08,759 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:30:08,764 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:30:09,134 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:30:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:57.131795+00:00 (in 47.997318 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:30:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:09 GMT)" (scheduled at 2025-11-04 09:30:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:30:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:30:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:30:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:30:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:30:09,484 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:30:11,251 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:30:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:30:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:15.535095+00:00 (in 4.004621 seconds) notificationworker stdout | 2025-11-04 09:30:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:21 GMT)" (scheduled at 2025-11-04 09:30:11.530064+00:00) notificationworker stdout | 2025-11-04 09:30:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:30:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 11, 530791), True, datetime.datetime(2025, 11, 4, 9, 30, 11, 530791), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:30:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:30:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:30:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:30:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:30:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:12.222828+00:00 (in 59.999599 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:30:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:12 GMT)" (scheduled at 2025-11-04 09:30:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:30:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:30:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:30:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:30:12,702 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:30:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:30:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:29.351216+00:00 (in 14.999627 seconds) quotatotalworker stdout | 2025-11-04 09:30:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:29 GMT)" (scheduled at 2025-11-04 09:30:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:30:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:30:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:30:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:30:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:30:15,200 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:30:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:30:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:17.479080+00:00 (in 1.998669 seconds) securityworker stdout | 2025-11-04 09:30:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:45 GMT)" (scheduled at 2025-11-04 09:30:15.479769+00:00) securityworker stdout | 2025-11-04 09:30:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:30:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:30:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:30:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:30:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 25, 15, 483914), 1, 2]) securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 25, 15, 483914), 1, 2]) securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:30:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:30:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:21.530064+00:00 (in 5.994512 seconds) notificationworker stdout | 2025-11-04 09:30:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:15 GMT)" (scheduled at 2025-11-04 09:30:15.535095+00:00) notificationworker stdout | 2025-11-04 09:30:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:30:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:30:15,698 [286] [DEBUG] [app] Starting request: urn:request:e89fc784-4c00-4dd8-8a36-72db7a1f4b66 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:15,699 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:15,701 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:15,702 [299] [DEBUG] [app] Starting request: urn:request:9dd47cc9-a4ce-4567-86b9-a3e2a7c4f5bd (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:15,703 [299] [DEBUG] [app] Ending request: urn:request:9dd47cc9-a4ce-4567-86b9-a3e2a7c4f5bd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9dd47cc9-a4ce-4567-86b9-a3e2a7c4f5bd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:30:15,703 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:30:15,703 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:15,704 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:15,706 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:15,707 [285] [DEBUG] [app] Starting request: urn:request:28fe2520-52e0-4537-b047-e48f374dfacc (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:15,707 [285] [DEBUG] [app] Ending request: urn:request:28fe2520-52e0-4537-b047-e48f374dfacc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:28fe2520-52e0-4537-b047-e48f374dfacc', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:30:15,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:30:15,707 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:30:15,708 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:15,708 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:15,708 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:15,716 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:15,716 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:15,726 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:15,729 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:15,732 [286] [DEBUG] [app] Ending request: urn:request:e89fc784-4c00-4dd8-8a36-72db7a1f4b66 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e89fc784-4c00-4dd8-8a36-72db7a1f4b66', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:15,732 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:30:15,732 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:30:15,783 [285] [DEBUG] [app] Starting request: urn:request:8ab31c7b-ad16-429f-942e-aa4f4391a0ef (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:15,788 [299] [DEBUG] [app] Starting request: urn:request:a559f3b2-b222-4e33-ac30-cb522bbbad46 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:15,788 [299] [DEBUG] [app] Ending request: urn:request:a559f3b2-b222-4e33-ac30-cb522bbbad46 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a559f3b2-b222-4e33-ac30-cb522bbbad46', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:30:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:30:15,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:15,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:15,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:15,793 [288] [DEBUG] [app] Starting request: urn:request:43d5d5b1-a1a2-4cca-a8f5-de01eadd05a9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:15,793 [288] [DEBUG] [app] Ending request: urn:request:43d5d5b1-a1a2-4cca-a8f5-de01eadd05a9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:43d5d5b1-a1a2-4cca-a8f5-de01eadd05a9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:30:15,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:30:15,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:30:15,794 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:15,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:15,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:15,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:15,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:15,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:15,815 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:15,818 [285] [DEBUG] [app] Ending request: urn:request:8ab31c7b-ad16-429f-942e-aa4f4391a0ef (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8ab31c7b-ad16-429f-942e-aa4f4391a0ef', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:15,818 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:30:15,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) proxycacheblobworker stdout | 2025-11-04 09:30:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:30:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:27.122849+00:00 (in 9.999614 seconds) proxycacheblobworker stdout | 2025-11-04 09:30:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:27 GMT)" (scheduled at 2025-11-04 09:30:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:30:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:30:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 17, 123501), True, datetime.datetime(2025, 11, 4, 9, 30, 17, 123501), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:30:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:30:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:30:17,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:27 GMT)" executed successfully builder stdout | 2025-11-04 09:30:17,251 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:30:17,264 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:30:17,264 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:30:17,274 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:30:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:30:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:45.479769+00:00 (in 28.000219 seconds) securityworker stdout | 2025-11-04 09:30:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:47 GMT)" (scheduled at 2025-11-04 09:30:17.479080+00:00) securityworker stdout | 2025-11-04 09:30:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:30:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:30:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:30:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:30:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:30:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:30:18,143 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:30:18,807 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:30:20,115 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:30:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:30:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:51.226294+00:00 (in 29.999547 seconds) autopruneworker stdout | 2025-11-04 09:30:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:51 GMT)" (scheduled at 2025-11-04 09:30:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:30:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248561235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:30:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:30:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:30:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:30:21,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:30:21,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:32.435813+00:00 (in 11.000625 seconds) namespacegcworker stdout | 2025-11-04 09:30:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:21 GMT)" (scheduled at 2025-11-04 09:30:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:30:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:30:21,436 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 21, 435453), True, datetime.datetime(2025, 11, 4, 9, 30, 21, 435453), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:30:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:30:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:30:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:30:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:30:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:31.530064+00:00 (in 9.999515 seconds) notificationworker stdout | 2025-11-04 09:30:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:31 GMT)" (scheduled at 2025-11-04 09:30:21.530064+00:00) notificationworker stdout | 2025-11-04 09:30:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:30:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 21, 530864), True, datetime.datetime(2025, 11, 4, 9, 30, 21, 530864), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:30:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:30:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:30:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:30:22,396 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:30:25,245 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:30:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:30:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:37.122849+00:00 (in 9.999574 seconds) proxycacheblobworker stdout | 2025-11-04 09:30:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:37 GMT)" (scheduled at 2025-11-04 09:30:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:30:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:30:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 27, 123607), True, datetime.datetime(2025, 11, 4, 9, 30, 27, 123607), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:30:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:30:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:30:27,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:30:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:30:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:57.222846+00:00 (in 29.999617 seconds) buildlogsarchiver stdout | 2025-11-04 09:30:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:57 GMT)" (scheduled at 2025-11-04 09:30:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:30:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 30, 27, 223422), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:30:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:30:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:30:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:30:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:30:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:30:57 GMT)" executed successfully builder stdout | 2025-11-04 09:30:27,274 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:30:27,288 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:30:27,288 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:30:27,299 [66] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:30:27,477 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:30:28,187 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:30:28,841 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:30:29,222 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:30:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:30:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:44.351216+00:00 (in 14.999608 seconds) quotatotalworker stdout | 2025-11-04 09:30:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:44 GMT)" (scheduled at 2025-11-04 09:30:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:30:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:30:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:30:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:30:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:30:29,625 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:29,872 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:30,062 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:30,077 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:30,161 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:30,697 [288] [DEBUG] [app] Starting request: urn:request:55e63de2-b602-45b4-bc13-606e4880a8a9 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:30,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:30,701 [297] [DEBUG] [app] Starting request: urn:request:90c36c2f-7dc3-42c9-9d4f-80256e020cb8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:30,702 [297] [DEBUG] [app] Ending request: urn:request:90c36c2f-7dc3-42c9-9d4f-80256e020cb8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:90c36c2f-7dc3-42c9-9d4f-80256e020cb8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:30:30,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:30:30,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:30,706 [285] [DEBUG] [app] Starting request: urn:request:97f38490-1dc3-4505-8682-841cda6b96e5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:30,706 [285] [DEBUG] [app] Ending request: urn:request:97f38490-1dc3-4505-8682-841cda6b96e5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:97f38490-1dc3-4505-8682-841cda6b96e5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:30:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:30:30,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:30,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:30,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:30,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:30,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:30,730 [288] [DEBUG] [app] Ending request: urn:request:55e63de2-b602-45b4-bc13-606e4880a8a9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:55e63de2-b602-45b4-bc13-606e4880a8a9', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:30,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:30:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:30:30,783 [285] [DEBUG] [app] Starting request: urn:request:7c1ee286-5617-427c-9c84-a147b3464cfb (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:30,786 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:30,787 [299] [DEBUG] [app] Starting request: urn:request:a07b4ac2-188a-4ffe-8428-b5916bef225c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:30,787 [299] [DEBUG] [app] Ending request: urn:request:a07b4ac2-188a-4ffe-8428-b5916bef225c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a07b4ac2-188a-4ffe-8428-b5916bef225c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:30:30,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:30:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:30,791 [287] [DEBUG] [app] Starting request: urn:request:d901e4bf-fe8c-461b-93ba-cd0f67497ffa (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:30,792 [287] [DEBUG] [app] Ending request: urn:request:d901e4bf-fe8c-461b-93ba-cd0f67497ffa (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d901e4bf-fe8c-461b-93ba-cd0f67497ffa', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:30:30,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:30:30,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:30,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:30,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:30,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:30,809 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:30,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:30,816 [285] [DEBUG] [app] Ending request: urn:request:7c1ee286-5617-427c-9c84-a147b3464cfb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7c1ee286-5617-427c-9c84-a147b3464cfb', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:30,816 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) gunicorn-web stdout | 2025-11-04 09:30:30,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:30:31,481 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:30:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:30:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:41.530064+00:00 (in 9.999549 seconds) notificationworker stdout | 2025-11-04 09:30:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:41 GMT)" (scheduled at 2025-11-04 09:30:31.530064+00:00) notificationworker stdout | 2025-11-04 09:30:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:30:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 31, 530785), True, datetime.datetime(2025, 11, 4, 9, 30, 31, 530785), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:30:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:30:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:30:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:30:32,276 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:30:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:30:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:21.434460+00:00 (in 48.998254 seconds) namespacegcworker stdout | 2025-11-04 09:30:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:32 GMT)" (scheduled at 2025-11-04 09:30:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:30:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:30:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:32 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:30:32,466 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:30:32,463 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:30:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:30:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:48.625192+00:00 (in 15.001846 seconds) exportactionlogsworker stdout | 2025-11-04 09:30:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:33 GMT)" (scheduled at 2025-11-04 09:30:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:30:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:30:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 33, 623608), True, datetime.datetime(2025, 11, 4, 9, 30, 33, 623608), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:30:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:30:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:30:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:30:33,753 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:30:33,754 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:30:33,857 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:30:34,634 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:30:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:30:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:38.437104+00:00 (in 3.000875 seconds) gcworker stdout | 2025-11-04 09:30:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:05 GMT)" (scheduled at 2025-11-04 09:30:35.435809+00:00) gcworker stdout | 2025-11-04 09:30:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039035436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:30:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:30:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:30:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:30:35,953 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:30:36,323 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:30:36,932 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:30:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:30:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:40.124445+00:00 (in 3.001160 seconds) proxycacheblobworker stdout | 2025-11-04 09:30:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:47 GMT)" (scheduled at 2025-11-04 09:30:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:30:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:30:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 37, 123584), True, datetime.datetime(2025, 11, 4, 9, 30, 37, 123584), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:30:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:30:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:30:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:47 GMT)" executed successfully builder stdout | 2025-11-04 09:30:37,299 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:30:37,312 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:30:37,312 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:30:37,330 [66] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:30:37,354 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:38,356 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:38,366 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:38,377 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:38,394 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:38,422 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:38,423 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:30:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:30:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:05.435809+00:00 (in 26.998347 seconds) gcworker stdout | 2025-11-04 09:30:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:08 GMT)" (scheduled at 2025-11-04 09:30:38.437104+00:00) gcworker stdout | 2025-11-04 09:30:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:30:38,447 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:30:38,448 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248578448, None, 1, 0]) gcworker stdout | 2025-11-04 09:30:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:30:38,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:30:38,604 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:30:38,611 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:30:38,767 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:30:38,769 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:30:38,774 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:30:39,493 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:30:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:30:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:01.922827+00:00 (in 21.997749 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:30:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:39 GMT)" (scheduled at 2025-11-04 09:30:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:30:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:30:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:30:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:30:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:47.122849+00:00 (in 6.997976 seconds) proxycacheblobworker stdout | 2025-11-04 09:30:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:40 GMT)" (scheduled at 2025-11-04 09:30:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:30:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:30:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:30:41,260 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:30:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:30:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:51.530064+00:00 (in 9.999600 seconds) notificationworker stdout | 2025-11-04 09:30:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:51 GMT)" (scheduled at 2025-11-04 09:30:41.530064+00:00) notificationworker stdout | 2025-11-04 09:30:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:30:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 41, 530762), True, datetime.datetime(2025, 11, 4, 9, 30, 41, 530762), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:30:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:30:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:30:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:30:42,711 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:30:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:30:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:57.523915+00:00 (in 14.090664 seconds) repositorygcworker stdout | 2025-11-04 09:30:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:43 GMT)" (scheduled at 2025-11-04 09:30:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:30:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:30:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 43, 433505), True, datetime.datetime(2025, 11, 4, 9, 30, 43, 433505), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:30:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:30:43,445 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:30:43,445 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:30:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:30:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:59.351216+00:00 (in 14.999600 seconds) quotatotalworker stdout | 2025-11-04 09:30:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:59 GMT)" (scheduled at 2025-11-04 09:30:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:30:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:30:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:30:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:30:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:30:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:30:45,208 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:30:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:30:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:47.479080+00:00 (in 1.998933 seconds) securityworker stdout | 2025-11-04 09:30:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:15 GMT)" (scheduled at 2025-11-04 09:30:45.479769+00:00) securityworker stdout | 2025-11-04 09:30:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:30:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:30:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:30:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:45,493 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:45,493 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:45,493 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:30:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:45,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 25, 45, 483543), 1, 2]) securityworker stdout | 2025-11-04 09:30:45,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:45,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:45,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:45,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:45,501 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:45,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 25, 45, 483543), 1, 2]) securityworker stdout | 2025-11-04 09:30:45,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:45,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:45,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:45,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:45,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:30:45,696 [285] [DEBUG] [app] Starting request: urn:request:8bcd9f79-65b0-4622-8f35-9a19f06ac76c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:45,701 [297] [DEBUG] [app] Starting request: urn:request:7a9f4bce-2e2f-4206-9741-b1d3b45cd389 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:45,701 [297] [DEBUG] [app] Ending request: urn:request:7a9f4bce-2e2f-4206-9741-b1d3b45cd389 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7a9f4bce-2e2f-4206-9741-b1d3b45cd389', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:30:45,701 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:30:45,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:45,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:45,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:45,705 [288] [DEBUG] [app] Starting request: urn:request:c10ee8a4-a242-4a3d-87c3-a36d76ba7d35 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:45,705 [288] [DEBUG] [app] Ending request: urn:request:c10ee8a4-a242-4a3d-87c3-a36d76ba7d35 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c10ee8a4-a242-4a3d-87c3-a36d76ba7d35', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:30:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:30:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:45,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:45,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:45,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:45,713 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:45,713 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:45,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:45,730 [285] [DEBUG] [app] Ending request: urn:request:8bcd9f79-65b0-4622-8f35-9a19f06ac76c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8bcd9f79-65b0-4622-8f35-9a19f06ac76c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:45,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:30:45,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:30:45,784 [287] [DEBUG] [app] Starting request: urn:request:49d9463a-c9ff-4182-a40a-819e893cbb98 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:30:45,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:45,788 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:30:45,788 [292] [DEBUG] [app] Starting request: urn:request:8f91fcf4-aa75-445d-9f9a-84445c59013f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:30:45,789 [292] [DEBUG] [app] Ending request: urn:request:8f91fcf4-aa75-445d-9f9a-84445c59013f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8f91fcf4-aa75-445d-9f9a-84445c59013f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:30:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:30:45,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:45,790 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:30:45,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:30:45,793 [285] [DEBUG] [app] Starting request: urn:request:a7d798db-11b5-479c-9f28-2377630e63b7 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:30:45,793 [285] [DEBUG] [app] Ending request: urn:request:a7d798db-11b5-479c-9f28-2377630e63b7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a7d798db-11b5-479c-9f28-2377630e63b7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:30:45,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:30:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:30:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:30:45,794 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:30:45,794 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:30:45,794 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:30:45,794 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:30:45,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:30:45,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:30:45,811 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:30:45,814 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:30:45,817 [287] [DEBUG] [app] Ending request: urn:request:49d9463a-c9ff-4182-a40a-819e893cbb98 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:49d9463a-c9ff-4182-a40a-819e893cbb98', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:30:45,817 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:30:45,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:30:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:30:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:30:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:30:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:30:57.122849+00:00 (in 9.999592 seconds) proxycacheblobworker stdout | 2025-11-04 09:30:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:57 GMT)" (scheduled at 2025-11-04 09:30:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:30:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:30:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 47, 123523), True, datetime.datetime(2025, 11, 4, 9, 30, 47, 123523), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:30:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:30:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:30:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:30:57 GMT)" executed successfully builder stdout | 2025-11-04 09:30:47,330 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:30:47,347 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:30:47,347 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:30:47,357 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:30:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:30:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:15.479769+00:00 (in 28.000303 seconds) securityworker stdout | 2025-11-04 09:30:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:17 GMT)" (scheduled at 2025-11-04 09:30:47.479080+00:00) securityworker stdout | 2025-11-04 09:30:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:30:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:30:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:30:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:30:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:30:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 25, 47, 483279), 1, 2]) securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:30:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:30:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:30:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:30:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 25, 47, 483279), 1, 2]) securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:30:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:30:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:30:48,152 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:30:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:30:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:33.622985+00:00 (in 44.997333 seconds) exportactionlogsworker stdout | 2025-11-04 09:30:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:48 GMT)" (scheduled at 2025-11-04 09:30:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:30:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:30:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:30:48,816 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:30:50,124 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:30:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:30:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:21.226294+00:00 (in 29.999565 seconds) autopruneworker stdout | 2025-11-04 09:30:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:21 GMT)" (scheduled at 2025-11-04 09:30:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:30:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248591234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:30:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:30:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:30:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:30:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:30:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:01.530064+00:00 (in 9.999538 seconds) notificationworker stdout | 2025-11-04 09:30:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:01 GMT)" (scheduled at 2025-11-04 09:30:51.530064+00:00) notificationworker stdout | 2025-11-04 09:30:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:30:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 51, 530775), True, datetime.datetime(2025, 11, 4, 9, 30, 51, 530775), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:30:51,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:30:51,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:30:51,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:30:52,405 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:30:55,253 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:30:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:30:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:07.122849+00:00 (in 9.999612 seconds) proxycacheblobworker stdout | 2025-11-04 09:30:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:07 GMT)" (scheduled at 2025-11-04 09:30:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:30:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:30:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 30, 57, 123568), True, datetime.datetime(2025, 11, 4, 9, 30, 57, 123568), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:30:57,132 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:30:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:09.133645+00:00 (in 12.001419 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:30:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:57 GMT)" (scheduled at 2025-11-04 09:30:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:30:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:30:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:30:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:30:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:30:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:30:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:30:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:30:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:30:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:27.222846+00:00 (in 29.999646 seconds) buildlogsarchiver stdout | 2025-11-04 09:30:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:27 GMT)" (scheduled at 2025-11-04 09:30:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:30:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 30, 57, 223390), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:30:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:30:57,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:30:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:30:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:30:57,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:27 GMT)" executed successfully builder stdout | 2025-11-04 09:30:57,357 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:30:57,371 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:30:57,371 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:30:57,381 [66] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:30:57,486 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:30:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:30:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:43.432867+00:00 (in 45.908555 seconds) repositorygcworker stdout | 2025-11-04 09:30:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:57 GMT)" (scheduled at 2025-11-04 09:30:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:30:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:30:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:31:57 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:30:58,196 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:30:58,849 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:30:59,231 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:30:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:30:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:14.351216+00:00 (in 14.999550 seconds) quotatotalworker stdout | 2025-11-04 09:30:59,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:14 GMT)" (scheduled at 2025-11-04 09:30:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:30:59,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:30:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:30:59,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:30:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:30:59,636 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:30:59,891 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:00,075 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:00,091 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:00,176 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:00,697 [285] [DEBUG] [app] Starting request: urn:request:6d0da8f8-79a9-4541-884a-20415cc88c5b (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:00,702 [299] [DEBUG] [app] Starting request: urn:request:420c0cd9-773a-4fad-9e32-4335dd80ba87 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:00,702 [299] [DEBUG] [app] Ending request: urn:request:420c0cd9-773a-4fad-9e32-4335dd80ba87 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:420c0cd9-773a-4fad-9e32-4335dd80ba87', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:31:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:31:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:00,706 [288] [DEBUG] [app] Starting request: urn:request:41159730-57ba-487b-9160-7153c63a4895 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:00,706 [288] [DEBUG] [app] Ending request: urn:request:41159730-57ba-487b-9160-7153c63a4895 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:41159730-57ba-487b-9160-7153c63a4895', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:31:00,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:31:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:00,731 [285] [DEBUG] [app] Ending request: urn:request:6d0da8f8-79a9-4541-884a-20415cc88c5b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6d0da8f8-79a9-4541-884a-20415cc88c5b', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:31:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:31:00,783 [285] [DEBUG] [app] Starting request: urn:request:3dc7e625-9c8e-4de7-a5eb-64b59b2e46ab (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:00,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:00,785 [292] [DEBUG] [app] Starting request: urn:request:b65d30a5-eb6c-4358-ad54-7fc85f69b05d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:00,785 [292] [DEBUG] [app] Ending request: urn:request:b65d30a5-eb6c-4358-ad54-7fc85f69b05d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b65d30a5-eb6c-4358-ad54-7fc85f69b05d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:31:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:31:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:00,789 [288] [DEBUG] [app] Starting request: urn:request:c923e8bd-01bb-4e6b-8779-507dc7c8f7f6 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:00,790 [288] [DEBUG] [app] Ending request: urn:request:c923e8bd-01bb-4e6b-8779-507dc7c8f7f6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c923e8bd-01bb-4e6b-8779-507dc7c8f7f6', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:31:00,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:31:00,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:00,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:00,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:00,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:00,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:00,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:00,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:00,813 [285] [DEBUG] [app] Ending request: urn:request:3dc7e625-9c8e-4de7-a5eb-64b59b2e46ab (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3dc7e625-9c8e-4de7-a5eb-64b59b2e46ab', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:00,813 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.030 47 0.031) gunicorn-web stdout | 2025-11-04 09:31:00,813 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:31:01,489 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:31:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:31:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:11.530064+00:00 (in 9.999586 seconds) notificationworker stdout | 2025-11-04 09:31:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:11 GMT)" (scheduled at 2025-11-04 09:31:01.530064+00:00) notificationworker stdout | 2025-11-04 09:31:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:31:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 1, 530764), True, datetime.datetime(2025, 11, 4, 9, 31, 1, 530764), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:31:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:31:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:31:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:31:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:31:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:39.924652+00:00 (in 38.001406 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:31:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:01 GMT)" (scheduled at 2025-11-04 09:31:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:31:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:31:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 1, 924046), True, datetime.datetime(2025, 11, 4, 9, 31, 1, 924046), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:31:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:31:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:31:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:31:02,285 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:31:02,474 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:31:02,483 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:31:03,762 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:31:03,762 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:31:03,866 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:31:04,644 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:31:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:31:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:08.437104+00:00 (in 3.000905 seconds) gcworker stdout | 2025-11-04 09:31:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:35 GMT)" (scheduled at 2025-11-04 09:31:05.435809+00:00) gcworker stdout | 2025-11-04 09:31:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039065436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:31:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:31:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:31:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:31:05,962 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:31:06,332 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:31:06,943 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:31:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:31:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:17.122849+00:00 (in 9.999578 seconds) proxycacheblobworker stdout | 2025-11-04 09:31:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:17 GMT)" (scheduled at 2025-11-04 09:31:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:31:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:31:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 7, 123564), True, datetime.datetime(2025, 11, 4, 9, 31, 7, 123564), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:31:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:31:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:31:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:31:07,363 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:31:07,381 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:31:07,394 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:31:07,394 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:31:07,404 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:31:08,379 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:08,368 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:08,387 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:08,407 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:08,434 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:08,436 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:31:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:31:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:35.435809+00:00 (in 26.998335 seconds) gcworker stdout | 2025-11-04 09:31:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:38 GMT)" (scheduled at 2025-11-04 09:31:08.437104+00:00) gcworker stdout | 2025-11-04 09:31:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:31:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248608449, None, 1, 0]) gcworker stdout | 2025-11-04 09:31:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:31:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:31:08,461 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:08,617 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:08,631 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:31:08,777 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:31:08,779 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:31:08,784 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:31:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:31:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:57.131795+00:00 (in 47.997715 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:31:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:09 GMT)" (scheduled at 2025-11-04 09:31:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:31:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:31:09,145 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:31:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:31:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:31:09,502 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:31:11,269 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:31:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:31:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:15.535095+00:00 (in 4.004567 seconds) notificationworker stdout | 2025-11-04 09:31:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:21 GMT)" (scheduled at 2025-11-04 09:31:11.530064+00:00) notificationworker stdout | 2025-11-04 09:31:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:31:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 11, 530832), True, datetime.datetime(2025, 11, 4, 9, 31, 11, 530832), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:31:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:31:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:31:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:31:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:31:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:12.222828+00:00 (in 59.999583 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:31:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:12 GMT)" (scheduled at 2025-11-04 09:31:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:31:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:31:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:31:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:31:12,720 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:31:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:31:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:29.351216+00:00 (in 14.999641 seconds) quotatotalworker stdout | 2025-11-04 09:31:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:29 GMT)" (scheduled at 2025-11-04 09:31:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:31:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:31:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:31:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:31:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:31:15,217 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:31:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:31:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:17.479080+00:00 (in 1.998881 seconds) securityworker stdout | 2025-11-04 09:31:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:45 GMT)" (scheduled at 2025-11-04 09:31:15.479769+00:00) securityworker stdout | 2025-11-04 09:31:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:31:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:31:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:31:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:31:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:15,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:15,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 26, 15, 483718), 1, 2]) securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:15,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 26, 15, 483718), 1, 2]) securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:31:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:31:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:21.530064+00:00 (in 5.994532 seconds) notificationworker stdout | 2025-11-04 09:31:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:15 GMT)" (scheduled at 2025-11-04 09:31:15.535095+00:00) notificationworker stdout | 2025-11-04 09:31:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:31:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:31:15,697 [285] [DEBUG] [app] Starting request: urn:request:dca23f81-c83f-4a3b-adeb-2f078360a492 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:15,701 [299] [DEBUG] [app] Starting request: urn:request:08ee1afc-cae7-4d0e-9d5d-e5810e1727ca (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:15,701 [299] [DEBUG] [app] Ending request: urn:request:08ee1afc-cae7-4d0e-9d5d-e5810e1727ca (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:08ee1afc-cae7-4d0e-9d5d-e5810e1727ca', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:31:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:31:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:15,705 [288] [DEBUG] [app] Starting request: urn:request:4d26f93d-9fde-4085-9b35-13cfeecc69e1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:15,706 [288] [DEBUG] [app] Ending request: urn:request:4d26f93d-9fde-4085-9b35-13cfeecc69e1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4d26f93d-9fde-4085-9b35-13cfeecc69e1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:31:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:31:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:15,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:15,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:15,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:15,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:15,731 [285] [DEBUG] [app] Ending request: urn:request:dca23f81-c83f-4a3b-adeb-2f078360a492 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:dca23f81-c83f-4a3b-adeb-2f078360a492', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:15,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:31:15,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:31:15,783 [285] [DEBUG] [app] Starting request: urn:request:6978c600-fd21-48f6-a091-729217a3ccc8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:15,785 [299] [DEBUG] [app] Starting request: urn:request:37ab10e7-145f-4882-b966-80e599c99dc8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:15,785 [299] [DEBUG] [app] Ending request: urn:request:37ab10e7-145f-4882-b966-80e599c99dc8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:37ab10e7-145f-4882-b966-80e599c99dc8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:31:15,785 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:31:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:15,789 [288] [DEBUG] [app] Starting request: urn:request:9ce4d2bf-93d5-4862-8ff0-4bc10cf89de5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:15,790 [288] [DEBUG] [app] Ending request: urn:request:9ce4d2bf-93d5-4862-8ff0-4bc10cf89de5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9ce4d2bf-93d5-4862-8ff0-4bc10cf89de5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:31:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:31:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:15,813 [285] [DEBUG] [app] Ending request: urn:request:6978c600-fd21-48f6-a091-729217a3ccc8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6978c600-fd21-48f6-a091-729217a3ccc8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:15,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:31:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.032 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:31:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:31:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:27.122849+00:00 (in 9.999575 seconds) proxycacheblobworker stdout | 2025-11-04 09:31:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:27 GMT)" (scheduled at 2025-11-04 09:31:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:31:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:31:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 17, 123541), True, datetime.datetime(2025, 11, 4, 9, 31, 17, 123541), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:31:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:31:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:31:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:27 GMT)" executed successfully builder stdout | 2025-11-04 09:31:17,404 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:31:17,418 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:31:17,418 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:31:17,428 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:31:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:31:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:45.479769+00:00 (in 28.000240 seconds) securityworker stdout | 2025-11-04 09:31:17,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:47 GMT)" (scheduled at 2025-11-04 09:31:17.479080+00:00) securityworker stdout | 2025-11-04 09:31:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:31:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:31:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:31:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:31:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:31:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:31:18,161 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:31:18,825 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:31:20,133 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:31:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:31:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:51.226294+00:00 (in 29.999534 seconds) autopruneworker stdout | 2025-11-04 09:31:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:51 GMT)" (scheduled at 2025-11-04 09:31:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:31:21,236 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248621236, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:31:21,241 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:31:21,241 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:31:21,241 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:31:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:31:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:32.435813+00:00 (in 11.000833 seconds) namespacegcworker stdout | 2025-11-04 09:31:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:21 GMT)" (scheduled at 2025-11-04 09:31:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:31:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:31:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 21, 435261), True, datetime.datetime(2025, 11, 4, 9, 31, 21, 435261), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:31:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:31:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:31:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:31:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:31:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:31.530064+00:00 (in 9.999623 seconds) notificationworker stdout | 2025-11-04 09:31:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:31 GMT)" (scheduled at 2025-11-04 09:31:21.530064+00:00) notificationworker stdout | 2025-11-04 09:31:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:31:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 21, 530708), True, datetime.datetime(2025, 11, 4, 9, 31, 21, 530708), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:31:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:31:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:31:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:31:22,413 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:23,189 [285] [DEBUG] [app] Starting request: urn:request:0d38f2e6-eb6c-443f-8d36-bb017d902f6e (/api/v1/user/notifications) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:31:23,189 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:31:23,189 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:31:23,190 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:31:23,201 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:31:23,201 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:31:23,201 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:31:23,202 [285] [DEBUG] [endpoints.api] Checking permission for user whuquay gunicorn-web stdout | 2025-11-04 09:31:23,202 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:31:23,202 [285] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:31:23,202 [285] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:31:23,202 [285] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:31:23,203 [285] [DEBUG] [peewee] ('(SELECT "t1"."id", "t1"."uuid", "t1"."kind_id", "t1"."metadata_json", "t1"."dismissed", "t1"."lookup_path", "t1"."created", "t1"."created" AS "cd", "t1"."target_id" FROM "notification" AS "t1" INNER JOIN "notificationkind" AS "t2" ON ("t1"."kind_id" = "t2"."id") WHERE (("t1"."dismissed" = %s) AND ("t1"."target_id" = %s))) UNION (SELECT "t3"."id", "t3"."uuid", "t3"."kind_id", "t3"."metadata_json", "t3"."dismissed", "t3"."lookup_path", "t3"."created", "t3"."created" AS "cd", "t3"."target_id" FROM "notification" AS "t3" INNER JOIN "notificationkind" AS "t4" ON ("t3"."kind_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t5"."id" = "t3"."target_id") INNER JOIN "team" AS "t6" ON ("t5"."id" = "t6"."organization_id") INNER JOIN "teamrole" AS "t7" ON ("t6"."role_id" = "t7"."id") INNER JOIN "teammember" AS "t8" ON ("t6"."id" = "t8"."team_id") INNER JOIN "user" AS "t9" ON ("t8"."user_id" = "t9"."id") WHERE (("t3"."dismissed" = %s) AND (("t9"."id" = %s) AND ("t7"."name" = %s)))) ORDER BY cd desc LIMIT %s', [False, 1, False, 1, 'admin', 6]) gunicorn-web stdout | 2025-11-04 09:31:23,209 [285] [DEBUG] [app] Ending request: urn:request:0d38f2e6-eb6c-443f-8d36-bb017d902f6e (/api/v1/user/notifications) {'endpoint': 'api.usernotificationlist', 'request_id': 'urn:request:0d38f2e6-eb6c-443f-8d36-bb017d902f6e', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/user/notifications', 'path': '/api/v1/user/notifications', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:31:23,209 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:31:23 +0000] "GET /api/v1/user/notifications HTTP/2.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.021 957 0.022) gunicorn-web stdout | 2025-11-04 09:31:23,210 [285] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:31:23 +0000] "GET /api/v1/user/notifications HTTP/1.0" 200 43 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" notificationworker stdout | 2025-11-04 09:31:25,262 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:31:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:31:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:37.122849+00:00 (in 9.999600 seconds) proxycacheblobworker stdout | 2025-11-04 09:31:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:37 GMT)" (scheduled at 2025-11-04 09:31:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:31:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:31:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 27, 123552), True, datetime.datetime(2025, 11, 4, 9, 31, 27, 123552), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:31:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:31:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:31:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:31:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:31:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:57.222846+00:00 (in 29.999642 seconds) buildlogsarchiver stdout | 2025-11-04 09:31:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:57 GMT)" (scheduled at 2025-11-04 09:31:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:31:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 31, 27, 223408), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:31:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:31:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:31:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:31:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:31:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:31:57 GMT)" executed successfully builder stdout | 2025-11-04 09:31:27,428 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:31:27,441 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:31:27,441 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:31:27,452 [66] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:31:27,495 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:31:28,205 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:31:28,858 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:31:29,240 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:31:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:31:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:44.351216+00:00 (in 14.999537 seconds) quotatotalworker stdout | 2025-11-04 09:31:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:44 GMT)" (scheduled at 2025-11-04 09:31:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:31:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:31:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:31:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:31:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:31:29,646 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:29,903 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:30,089 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:30,103 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:30,190 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:30,697 [287] [DEBUG] [app] Starting request: urn:request:d9f03cc9-b367-4612-983d-11a8a8fb9e5c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:30,697 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:30,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:30,701 [299] [DEBUG] [app] Starting request: urn:request:1bce2096-efb5-4dbe-9729-0bf5ed53b37c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:30,701 [299] [DEBUG] [app] Ending request: urn:request:1bce2096-efb5-4dbe-9729-0bf5ed53b37c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1bce2096-efb5-4dbe-9729-0bf5ed53b37c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:31:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:31:30,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:30,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:30,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:30,706 [285] [DEBUG] [app] Starting request: urn:request:c0c9ea1a-b337-4128-9278-0753cd534516 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:30,706 [285] [DEBUG] [app] Ending request: urn:request:c0c9ea1a-b337-4128-9278-0753cd534516 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c0c9ea1a-b337-4128-9278-0753cd534516', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:31:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:31:30,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:30,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:30,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:30,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:30,715 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:30,715 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:30,724 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:30,727 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:30,730 [287] [DEBUG] [app] Ending request: urn:request:d9f03cc9-b367-4612-983d-11a8a8fb9e5c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d9f03cc9-b367-4612-983d-11a8a8fb9e5c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:30,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:31:30,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:31:30,784 [285] [DEBUG] [app] Starting request: urn:request:fdb55a10-a219-4941-a22e-7411f479d626 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:30,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:30,788 [299] [DEBUG] [app] Starting request: urn:request:8a7d02a2-2bb5-43f4-8e9e-5799247a9e7e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:30,788 [299] [DEBUG] [app] Ending request: urn:request:8a7d02a2-2bb5-43f4-8e9e-5799247a9e7e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8a7d02a2-2bb5-43f4-8e9e-5799247a9e7e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:31:30,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:31:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:30,792 [286] [DEBUG] [app] Starting request: urn:request:6fe0a1ae-a49d-44f5-9702-fa224637e2c4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:30,792 [286] [DEBUG] [app] Ending request: urn:request:6fe0a1ae-a49d-44f5-9702-fa224637e2c4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6fe0a1ae-a49d-44f5-9702-fa224637e2c4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:31:30,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:31:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:30,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:30,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:30,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:30,817 [285] [DEBUG] [app] Ending request: urn:request:fdb55a10-a219-4941-a22e-7411f479d626 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fdb55a10-a219-4941-a22e-7411f479d626', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:30,817 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:31:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" manifestsubjectbackfillworker stdout | 2025-11-04 09:31:31,498 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:31:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:31:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:41.530064+00:00 (in 9.999548 seconds) notificationworker stdout | 2025-11-04 09:31:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:41 GMT)" (scheduled at 2025-11-04 09:31:31.530064+00:00) notificationworker stdout | 2025-11-04 09:31:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:31:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 31, 530810), True, datetime.datetime(2025, 11, 4, 9, 31, 31, 530810), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:31:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:31:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:31:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:31:32,294 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:31:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:31:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:21.434460+00:00 (in 48.998272 seconds) namespacegcworker stdout | 2025-11-04 09:31:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:32 GMT)" (scheduled at 2025-11-04 09:31:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:31:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:31:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:32 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:31:32,483 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:31:32,491 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:31:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:31:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:48.625192+00:00 (in 15.001786 seconds) exportactionlogsworker stdout | 2025-11-04 09:31:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:33 GMT)" (scheduled at 2025-11-04 09:31:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:31:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:31:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 33, 623944), True, datetime.datetime(2025, 11, 4, 9, 31, 33, 623944), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:31:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:31:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:31:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:31:33,770 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:31:33,771 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:31:33,874 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:31:34,653 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:31:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:31:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:38.437104+00:00 (in 3.000873 seconds) gcworker stdout | 2025-11-04 09:31:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:05 GMT)" (scheduled at 2025-11-04 09:31:35.435809+00:00) gcworker stdout | 2025-11-04 09:31:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039095436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:31:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:31:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:31:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:31:35,972 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:31:36,341 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:31:36,953 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:31:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:31:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:40.124445+00:00 (in 3.001159 seconds) proxycacheblobworker stdout | 2025-11-04 09:31:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:47 GMT)" (scheduled at 2025-11-04 09:31:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:31:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:31:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 37, 123579), True, datetime.datetime(2025, 11, 4, 9, 31, 37, 123579), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:31:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:31:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:31:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:31:37,377 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:31:37,452 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:31:37,466 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:31:37,466 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:31:37,476 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:31:38,391 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:38,391 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:38,402 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:38,419 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:31:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:31:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:05.435809+00:00 (in 26.998340 seconds) gcworker stdout | 2025-11-04 09:31:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:08 GMT)" (scheduled at 2025-11-04 09:31:38.437104+00:00) gcworker stdout | 2025-11-04 09:31:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:31:38,446 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:38,447 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:31:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248638449, None, 1, 0]) gcworker stdout | 2025-11-04 09:31:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:31:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:31:38,475 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:38,635 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:31:38,643 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:31:38,794 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:31:38,787 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:31:38,789 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:31:38,926 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run globalpromstats stdout | 2025-11-04 09:31:38,926 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 10:31:38.926121+00:00 (in 3599.999536 seconds) globalpromstats stdout | 2025-11-04 09:31:38,926 [73] [INFO] [apscheduler.executors.default] Running job "GlobalPrometheusStatsWorker._try_report_stats (trigger: interval[1:00:00], next run at: 2025-11-04 10:31:38 GMT)" (scheduled at 2025-11-04 09:31:38.926121+00:00) globalpromstats stdout | 2025-11-04 09:31:38,926 [73] [DEBUG] [__main__] Attempting to report stats globalpromstats stdout | 2025-11-04 09:31:38,926 [73] [DEBUG] [util.locking] Acquiring global lock GLOBAL_PROM_STATS globalpromstats stdout | 2025-11-04 09:31:38,926 [73] [DEBUG] [redis_lock.acquire] Getting 'lock:GLOBAL_PROM_STATS' ... globalpromstats stdout | 2025-11-04 09:31:38,927 [73] [INFO] [redis_lock.acquire] Got lock for 'lock:GLOBAL_PROM_STATS'. globalpromstats stdout | 2025-11-04 09:31:38,927 [73] [DEBUG] [util.locking] Acquired lock GLOBAL_PROM_STATS globalpromstats stdout | 2025-11-04 09:31:38,927 [73] [DEBUG] [__main__] Reporting global stats globalpromstats stdout | 2025-11-04 09:31:38,927 [73] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "repository" AS "t1") AS "_wrapped"', []) globalpromstats stdout | 2025-11-04 09:31:38,939 [73] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."organization" = %s) AND ("t1"."robot" = %s)) AND ("t1"."id" NOT IN (SELECT "t2"."namespace_id" FROM "deletednamespace" AS "t2")))) AS "_wrapped"', [False, False]) globalpromstats stdout | 2025-11-04 09:31:38,943 [73] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."organization" = %s) AND ("t1"."robot" = %s)) AND ("t1"."enabled" = %s))) AS "_wrapped"', [True, False, True]) globalpromstats stdout | 2025-11-04 09:31:38,946 [73] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "robotaccounttoken" AS "t1") AS "_wrapped"', []) globalpromstats stdout | 2025-11-04 09:31:38,949 [73] [DEBUG] [data.database] Disconnecting from database. globalpromstats stdout | 2025-11-04 09:31:38,949 [73] [DEBUG] [util.locking] Releasing lock GLOBAL_PROM_STATS globalpromstats stdout | 2025-11-04 09:31:38,949 [73] [DEBUG] [redis_lock.release] Releasing 'lock:GLOBAL_PROM_STATS'. globalpromstats stdout | 2025-11-04 09:31:38,949 [73] [DEBUG] [util.locking] Released lock GLOBAL_PROM_STATS globalpromstats stdout | 2025-11-04 09:31:38,949 [73] [INFO] [apscheduler.executors.default] Job "GlobalPrometheusStatsWorker._try_report_stats (trigger: interval[1:00:00], next run at: 2025-11-04 10:31:38 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:31:39,511 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:31:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:31:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:01.922827+00:00 (in 21.997774 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:31:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:39 GMT)" (scheduled at 2025-11-04 09:31:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:31:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:31:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:31:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:31:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:47.122849+00:00 (in 6.997997 seconds) proxycacheblobworker stdout | 2025-11-04 09:31:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:40 GMT)" (scheduled at 2025-11-04 09:31:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:31:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:31:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:31:41,277 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:31:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:31:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:51.530064+00:00 (in 9.999501 seconds) notificationworker stdout | 2025-11-04 09:31:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:51 GMT)" (scheduled at 2025-11-04 09:31:41.530064+00:00) notificationworker stdout | 2025-11-04 09:31:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:31:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 41, 530883), True, datetime.datetime(2025, 11, 4, 9, 31, 41, 530883), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:31:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:31:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:31:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:31:42,729 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:31:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:31:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:57.523915+00:00 (in 14.090675 seconds) repositorygcworker stdout | 2025-11-04 09:31:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:43 GMT)" (scheduled at 2025-11-04 09:31:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:31:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:31:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 43, 433466), True, datetime.datetime(2025, 11, 4, 9, 31, 43, 433466), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:31:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:31:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:31:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:31:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:31:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:59.351216+00:00 (in 14.999578 seconds) quotatotalworker stdout | 2025-11-04 09:31:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:59 GMT)" (scheduled at 2025-11-04 09:31:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:31:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:31:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:31:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:31:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:31:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:31:45,226 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:31:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:31:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:47.479080+00:00 (in 1.998874 seconds) securityworker stdout | 2025-11-04 09:31:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:15 GMT)" (scheduled at 2025-11-04 09:31:45.479769+00:00) securityworker stdout | 2025-11-04 09:31:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:31:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:31:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:31:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 26, 45, 483740), 1, 2]) securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 26, 45, 483740), 1, 2]) securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:31:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:15 GMT)" executed successfully securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker gunicorn-web stdout | 2025-11-04 09:31:45,697 [288] [DEBUG] [app] Starting request: urn:request:8b3b8761-611a-417e-a7c2-967b640425cc (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:45,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:45,701 [292] [DEBUG] [app] Starting request: urn:request:1dc3a11b-9f16-4a17-9cf1-c66b031c1fc3 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:45,702 [292] [DEBUG] [app] Ending request: urn:request:1dc3a11b-9f16-4a17-9cf1-c66b031c1fc3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1dc3a11b-9f16-4a17-9cf1-c66b031c1fc3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:31:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:31:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:45,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:45,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:45,706 [286] [DEBUG] [app] Starting request: urn:request:f094d2c9-1786-49d2-aedc-9a5932ecf2ee (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:45,706 [286] [DEBUG] [app] Ending request: urn:request:f094d2c9-1786-49d2-aedc-9a5932ecf2ee (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f094d2c9-1786-49d2-aedc-9a5932ecf2ee', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:31:45,706 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:31:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:45,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:45,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:45,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:45,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:45,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:45,730 [288] [DEBUG] [app] Ending request: urn:request:8b3b8761-611a-417e-a7c2-967b640425cc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8b3b8761-611a-417e-a7c2-967b640425cc', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:45,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:31:45,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:31:45,784 [285] [DEBUG] [app] Starting request: urn:request:da0714f0-8813-487e-af5d-59a5bbc63415 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:31:45,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:31:45,788 [292] [DEBUG] [app] Starting request: urn:request:bb6c35ae-362d-44ab-b058-9dc4b17b017b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:31:45,788 [292] [DEBUG] [app] Ending request: urn:request:bb6c35ae-362d-44ab-b058-9dc4b17b017b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bb6c35ae-362d-44ab-b058-9dc4b17b017b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:31:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:31:45,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:45,790 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:31:45,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:31:45,793 [286] [DEBUG] [app] Starting request: urn:request:4f18cfe2-fd67-42fe-aaea-660699fc11e8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:31:45,793 [286] [DEBUG] [app] Ending request: urn:request:4f18cfe2-fd67-42fe-aaea-660699fc11e8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4f18cfe2-fd67-42fe-aaea-660699fc11e8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:31:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:31:45,793 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:31:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:31:45,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:31:45,794 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:31:45,794 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:31:45,794 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:31:45,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:31:45,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:31:45,811 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:31:45,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:31:45,817 [285] [DEBUG] [app] Ending request: urn:request:da0714f0-8813-487e-af5d-59a5bbc63415 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:da0714f0-8813-487e-af5d-59a5bbc63415', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:31:45,818 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:31:45,818 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:31:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:31:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:31:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:31:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:57.122849+00:00 (in 9.999567 seconds) proxycacheblobworker stdout | 2025-11-04 09:31:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:57 GMT)" (scheduled at 2025-11-04 09:31:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:31:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:31:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 47, 123618), True, datetime.datetime(2025, 11, 4, 9, 31, 47, 123618), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:31:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:31:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:31:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:31:57 GMT)" executed successfully builder stdout | 2025-11-04 09:31:47,476 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue securityworker stdout | 2025-11-04 09:31:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:31:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:15.479769+00:00 (in 28.000302 seconds) securityworker stdout | 2025-11-04 09:31:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:17 GMT)" (scheduled at 2025-11-04 09:31:47.479080+00:00) securityworker stdout | 2025-11-04 09:31:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:31:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:31:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:31:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) builder stdout | 2025-11-04 09:31:47,490 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:31:47,490 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics securityworker stdout | 2025-11-04 09:31:47,493 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:47,496 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:47,496 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:31:47,496 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:47,496 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:47,497 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) builder stdout | 2025-11-04 09:31:47,500 [66] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:47,501 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 26, 47, 482848), 1, 2]) securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:31:47,505 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:31:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 26, 47, 482848), 1, 2]) securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:47,509 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:31:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:31:47,509 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:31:48,170 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:31:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:31:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:31:58.624878+00:00 (in 9.999204 seconds) exportactionlogsworker stdout | 2025-11-04 09:31:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:48 GMT)" (scheduled at 2025-11-04 09:31:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:31:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:31:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:31:48,834 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:31:50,142 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:31:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:31:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:21.226294+00:00 (in 29.999534 seconds) autopruneworker stdout | 2025-11-04 09:31:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:21 GMT)" (scheduled at 2025-11-04 09:31:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:31:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248651234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:31:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:31:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:31:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:31:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:31:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:01.530064+00:00 (in 9.999511 seconds) notificationworker stdout | 2025-11-04 09:31:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:01 GMT)" (scheduled at 2025-11-04 09:31:51.530064+00:00) notificationworker stdout | 2025-11-04 09:31:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:31:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 51, 530870), True, datetime.datetime(2025, 11, 4, 9, 31, 51, 530870), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:31:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:31:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:31:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:31:52,422 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:31:55,280 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:31:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:31:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:07.122849+00:00 (in 9.999564 seconds) proxycacheblobworker stdout | 2025-11-04 09:31:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:07 GMT)" (scheduled at 2025-11-04 09:31:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:31:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:31:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 31, 57, 123571), True, datetime.datetime(2025, 11, 4, 9, 31, 57, 123571), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:31:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:31:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:09.133645+00:00 (in 12.001471 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:31:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:57 GMT)" (scheduled at 2025-11-04 09:31:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:31:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:31:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:31:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:31:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:31:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:31:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:31:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:31:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:31:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:27.222846+00:00 (in 29.999603 seconds) buildlogsarchiver stdout | 2025-11-04 09:31:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:27 GMT)" (scheduled at 2025-11-04 09:31:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:31:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 31, 57, 223469), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:31:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:31:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:31:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:31:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:31:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:27 GMT)" executed successfully builder stdout | 2025-11-04 09:31:57,501 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue quotaregistrysizeworker stdout | 2025-11-04 09:31:57,504 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:31:57,514 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:31:57,514 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics repositorygcworker stdout | 2025-11-04 09:31:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:31:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:43.432867+00:00 (in 45.908525 seconds) repositorygcworker stdout | 2025-11-04 09:31:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:57 GMT)" (scheduled at 2025-11-04 09:31:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:31:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:31:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:32:57 GMT)" executed successfully builder stdout | 2025-11-04 09:31:57,524 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:31:58,224 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:31:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:31:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:33.622985+00:00 (in 34.997715 seconds) exportactionlogsworker stdout | 2025-11-04 09:31:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:36:58 GMT)" (scheduled at 2025-11-04 09:31:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:31:58,625 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 31, 58, 625512), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:31:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 31, 58, 625512), True, datetime.datetime(2025, 11, 4, 9, 31, 58, 625512), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:31:58,641 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 31, 58, 625512), True, datetime.datetime(2025, 11, 4, 9, 31, 58, 625512), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 31, 58, 625512), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:31:58,644 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:31:58,644 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:36:58 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:31:58,877 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:31:59,260 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:31:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:31:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:14.351216+00:00 (in 14.999583 seconds) quotatotalworker stdout | 2025-11-04 09:31:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:14 GMT)" (scheduled at 2025-11-04 09:31:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:31:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:31:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:31:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:31:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:31:59,656 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:31:59,915 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:00,102 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:00,117 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:00,205 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:00,697 [285] [DEBUG] [app] Starting request: urn:request:893164c0-db2e-437a-877a-3ca51173e84e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:00,701 [292] [DEBUG] [app] Starting request: urn:request:b0a9550f-8151-4999-bd55-20ffd299c7ef (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:00,702 [292] [DEBUG] [app] Ending request: urn:request:b0a9550f-8151-4999-bd55-20ffd299c7ef (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b0a9550f-8151-4999-bd55-20ffd299c7ef', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:32:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:00,706 [287] [DEBUG] [app] Starting request: urn:request:017bb2c0-c440-4023-8b93-d3a1fc06beca (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:00,706 [287] [DEBUG] [app] Ending request: urn:request:017bb2c0-c440-4023-8b93-d3a1fc06beca (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:017bb2c0-c440-4023-8b93-d3a1fc06beca', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:32:00,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:32:00,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:00,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:00,730 [285] [DEBUG] [app] Ending request: urn:request:893164c0-db2e-437a-877a-3ca51173e84e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:893164c0-db2e-437a-877a-3ca51173e84e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:00,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:00,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:32:00,783 [285] [DEBUG] [app] Starting request: urn:request:1478ef84-e977-4a55-8c1b-5dfe4d3487b8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:00,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:00,784 [292] [DEBUG] [app] Starting request: urn:request:99675661-b79d-46cc-8a81-5009c1cc0812 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:00,785 [292] [DEBUG] [app] Ending request: urn:request:99675661-b79d-46cc-8a81-5009c1cc0812 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:99675661-b79d-46cc-8a81-5009c1cc0812', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:00,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:32:00,785 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:00,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:00,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:00,789 [287] [DEBUG] [app] Starting request: urn:request:29155f36-0e91-472b-8c73-c4754eea3d65 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:00,789 [287] [DEBUG] [app] Ending request: urn:request:29155f36-0e91-472b-8c73-c4754eea3d65 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:29155f36-0e91-472b-8c73-c4754eea3d65', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:32:00,789 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:32:00,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:00,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:00,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:00,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:00,797 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:00,797 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:00,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:00,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:00,813 [285] [DEBUG] [app] Ending request: urn:request:1478ef84-e977-4a55-8c1b-5dfe4d3487b8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1478ef84-e977-4a55-8c1b-5dfe4d3487b8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:00,813 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:00,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) manifestsubjectbackfillworker stdout | 2025-11-04 09:32:01,506 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:32:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:32:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:11.530064+00:00 (in 9.999524 seconds) notificationworker stdout | 2025-11-04 09:32:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:11 GMT)" (scheduled at 2025-11-04 09:32:01.530064+00:00) notificationworker stdout | 2025-11-04 09:32:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:32:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 1, 530813), True, datetime.datetime(2025, 11, 4, 9, 32, 1, 530813), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:32:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:32:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:32:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:11 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:32:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:32:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:21.924304+00:00 (in 20.001090 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:32:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:01 GMT)" (scheduled at 2025-11-04 09:32:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:32:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:32:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 1, 923443), True, datetime.datetime(2025, 11, 4, 9, 32, 1, 923443), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:32:01,937 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:32:01,937 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:32:01,937 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:32:02,303 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:32:02,492 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:32:02,500 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:32:03,779 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:32:03,790 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:32:03,883 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:32:04,663 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:32:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:32:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:08.437104+00:00 (in 3.000787 seconds) gcworker stdout | 2025-11-04 09:32:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:35 GMT)" (scheduled at 2025-11-04 09:32:05.435809+00:00) gcworker stdout | 2025-11-04 09:32:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039125436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:32:05,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:32:05,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:32:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:32:05,981 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:32:06,350 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:32:06,968 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:32:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:32:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:17.122849+00:00 (in 9.999602 seconds) proxycacheblobworker stdout | 2025-11-04 09:32:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:17 GMT)" (scheduled at 2025-11-04 09:32:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:32:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:32:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 7, 123525), True, datetime.datetime(2025, 11, 4, 9, 32, 7, 123525), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:32:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:32:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:32:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:32:07,391 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:32:07,525 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:32:07,538 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:32:07,538 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:32:07,549 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:32:08,402 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:08,402 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:08,416 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:08,432 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:32:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:32:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:35.435809+00:00 (in 26.998306 seconds) gcworker stdout | 2025-11-04 09:32:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:38 GMT)" (scheduled at 2025-11-04 09:32:08.437104+00:00) gcworker stdout | 2025-11-04 09:32:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:32:08,448 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248668448, None, 1, 0]) gcworker stdout | 2025-11-04 09:32:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:32:08,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:32:08,456 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:08,459 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:08,500 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:08,647 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:08,655 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:32:08,804 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:32:08,812 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:32:08,813 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:32:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:32:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:57.131795+00:00 (in 47.997768 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:32:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:09 GMT)" (scheduled at 2025-11-04 09:32:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:32:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:32:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:32:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:32:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:32:09,520 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:32:11,287 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:32:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:32:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:15.535095+00:00 (in 4.004642 seconds) notificationworker stdout | 2025-11-04 09:32:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:21 GMT)" (scheduled at 2025-11-04 09:32:11.530064+00:00) notificationworker stdout | 2025-11-04 09:32:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:32:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 11, 530745), True, datetime.datetime(2025, 11, 4, 9, 32, 11, 530745), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:32:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:32:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:32:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:32:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:32:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:12.222828+00:00 (in 59.999579 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:32:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:12 GMT)" (scheduled at 2025-11-04 09:32:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:32:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:32:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:32:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:32:12,738 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:32:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:32:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:29.351216+00:00 (in 14.999593 seconds) quotatotalworker stdout | 2025-11-04 09:32:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:29 GMT)" (scheduled at 2025-11-04 09:32:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:32:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:32:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:32:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:32:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:32:15,234 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:32:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:32:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:17.479080+00:00 (in 1.998880 seconds) securityworker stdout | 2025-11-04 09:32:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:45 GMT)" (scheduled at 2025-11-04 09:32:15.479769+00:00) securityworker stdout | 2025-11-04 09:32:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:32:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:32:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:32:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 27, 15, 483908), 1, 2]) securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 27, 15, 483908), 1, 2]) securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:32:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:32:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:32:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:21.530064+00:00 (in 5.994497 seconds) notificationworker stdout | 2025-11-04 09:32:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:15 GMT)" (scheduled at 2025-11-04 09:32:15.535095+00:00) notificationworker stdout | 2025-11-04 09:32:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:32:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:32:15,696 [285] [DEBUG] [app] Starting request: urn:request:409dd8e7-c54b-4272-9257-2a3fcf068618 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:15,701 [292] [DEBUG] [app] Starting request: urn:request:f480fab7-b890-4d58-85bf-771cc094725c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:15,701 [292] [DEBUG] [app] Ending request: urn:request:f480fab7-b890-4d58-85bf-771cc094725c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f480fab7-b890-4d58-85bf-771cc094725c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:15,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:32:15,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:15,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:15,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:15,705 [288] [DEBUG] [app] Starting request: urn:request:897f2a15-42df-4173-9e06-351d3fa9c9c5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:15,705 [288] [DEBUG] [app] Ending request: urn:request:897f2a15-42df-4173-9e06-351d3fa9c9c5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:897f2a15-42df-4173-9e06-351d3fa9c9c5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:32:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:32:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:15,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:15,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:15,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:15,730 [285] [DEBUG] [app] Ending request: urn:request:409dd8e7-c54b-4272-9257-2a3fcf068618 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:409dd8e7-c54b-4272-9257-2a3fcf068618', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:15,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:32:15,783 [285] [DEBUG] [app] Starting request: urn:request:82bedb22-925d-4ddf-9c5c-8137688e75d8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:15,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:15,785 [292] [DEBUG] [app] Starting request: urn:request:fff0e54e-c7a6-4703-96f4-3fa7c2e77aba (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:15,785 [292] [DEBUG] [app] Ending request: urn:request:fff0e54e-c7a6-4703-96f4-3fa7c2e77aba (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fff0e54e-c7a6-4703-96f4-3fa7c2e77aba', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:15,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:32:15,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:15,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:15,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:15,789 [288] [DEBUG] [app] Starting request: urn:request:4cd56e6d-a57b-41f2-a536-5b2e8abe076e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:15,790 [288] [DEBUG] [app] Ending request: urn:request:4cd56e6d-a57b-41f2-a536-5b2e8abe076e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4cd56e6d-a57b-41f2-a536-5b2e8abe076e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:32:15,790 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:32:15,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:15,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:15,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:15,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:15,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:15,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:15,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:15,810 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:15,813 [285] [DEBUG] [app] Ending request: urn:request:82bedb22-925d-4ddf-9c5c-8137688e75d8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:82bedb22-925d-4ddf-9c5c-8137688e75d8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:15,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:15,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:32:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:32:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:27.122849+00:00 (in 9.999637 seconds) proxycacheblobworker stdout | 2025-11-04 09:32:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:27 GMT)" (scheduled at 2025-11-04 09:32:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:32:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:32:17,123 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 17, 123440), True, datetime.datetime(2025, 11, 4, 9, 32, 17, 123440), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:32:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:32:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:32:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:32:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:32:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:45.479769+00:00 (in 28.000231 seconds) securityworker stdout | 2025-11-04 09:32:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:47 GMT)" (scheduled at 2025-11-04 09:32:17.479080+00:00) securityworker stdout | 2025-11-04 09:32:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:32:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:32:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:32:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:32:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:32:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:47 GMT)" executed successfully builder stdout | 2025-11-04 09:32:17,549 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:32:17,561 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:32:17,561 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:32:17,571 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:32:18,179 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:32:18,853 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:32:20,162 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:32:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:32:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:51.226294+00:00 (in 29.999538 seconds) autopruneworker stdout | 2025-11-04 09:32:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:51 GMT)" (scheduled at 2025-11-04 09:32:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:32:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248681235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:32:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:32:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:32:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:32:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:32:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:32.435813+00:00 (in 11.000967 seconds) namespacegcworker stdout | 2025-11-04 09:32:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:21 GMT)" (scheduled at 2025-11-04 09:32:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:32:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:32:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 21, 435174), True, datetime.datetime(2025, 11, 4, 9, 32, 21, 435174), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:32:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:32:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:32:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:32:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:32:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:31.530064+00:00 (in 9.999610 seconds) notificationworker stdout | 2025-11-04 09:32:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:31 GMT)" (scheduled at 2025-11-04 09:32:21.530064+00:00) notificationworker stdout | 2025-11-04 09:32:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:32:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 21, 530761), True, datetime.datetime(2025, 11, 4, 9, 32, 21, 530761), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:32:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:32:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:32:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:31 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:32:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:32:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:39.924652+00:00 (in 17.999940 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:32:21,924 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:37:21 GMT)" (scheduled at 2025-11-04 09:32:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:32:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 32, 21, 924951), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:32:21,938 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 32, 21, 924951), True, datetime.datetime(2025, 11, 4, 9, 32, 21, 924951), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:32:21,952 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 32, 21, 924951), True, datetime.datetime(2025, 11, 4, 9, 32, 21, 924951), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 32, 21, 924951), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:32:21,955 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:32:21,955 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:37:21 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:32:22,431 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:32:25,289 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:32:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:32:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:37.122849+00:00 (in 9.999586 seconds) proxycacheblobworker stdout | 2025-11-04 09:32:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:37 GMT)" (scheduled at 2025-11-04 09:32:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:32:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:32:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 27, 123598), True, datetime.datetime(2025, 11, 4, 9, 32, 27, 123598), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:32:27,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:32:27,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:32:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:32:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:32:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:57.222846+00:00 (in 29.999566 seconds) buildlogsarchiver stdout | 2025-11-04 09:32:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:57 GMT)" (scheduled at 2025-11-04 09:32:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:32:27,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 32, 27, 223552), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:32:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:32:27,239 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:32:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:32:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:32:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:32:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:32:27,513 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:32:27,572 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:32:27,585 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:32:27,585 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:32:27,595 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:32:28,233 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:32:28,886 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:32:29,269 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:32:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:32:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:44.351216+00:00 (in 14.999535 seconds) quotatotalworker stdout | 2025-11-04 09:32:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:44 GMT)" (scheduled at 2025-11-04 09:32:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:32:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:32:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:32:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:32:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:32:29,666 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:29,927 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:30,115 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:30,130 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:30,220 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:30,575 [286] [DEBUG] [app] Starting request: urn:request:8d72bbee-602f-48a3-a098-abc1f348d446 (/api/v1/superuser/users/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:30,575 [285] [DEBUG] [app] Starting request: urn:request:0569ae39-331f-4102-9a4a-dcd013c74e61 (/api/v1/organization/build_test) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:30,575 [286] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,575 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,575 [286] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,575 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,576 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,576 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,587 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:30,587 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:30,587 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,587 [286] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:30,587 [286] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:30,587 [286] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,588 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."organization" = %s) AND ("t1"."username" = %s)) LIMIT %s OFFSET %s', [True, 'build_test', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,588 [286] [DEBUG] [endpoints.api] Checking fresh login for user whuquay: Last login at 2025-11-04 07:15:43+00:00 gunicorn-web stdout | 2025-11-04 09:32:30,588 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['whuquay', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,591 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,591 [285] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:30,591 [285] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:30,591 [285] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:30,592 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."organization_id", "t1"."role_id", "t1"."description", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed", "t3"."id", "t3"."name" FROM "team" AS "t1" INNER JOIN "teamrole" AS "t3" ON ("t1"."role_id" = "t3"."id") INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") INNER JOIN "teammember" AS "t4" ON ("t4"."team_id" = "t1"."id") INNER JOIN "user" AS "t5" ON ("t4"."user_id" = "t5"."id") WHERE (("t5"."id" = %s) AND ("t2"."organization" = %s))', [1, True]) gunicorn-web stdout | 2025-11-04 09:32:30,592 [286] [DEBUG] [app] Ending request: urn:request:8d72bbee-602f-48a3-a098-abc1f348d446 (/api/v1/superuser/users/) {'endpoint': 'api.superuserlist', 'request_id': 'urn:request:8d72bbee-602f-48a3-a098-abc1f348d446', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/superuser/users/', 'path': '/api/v1/superuser/users/', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:30,593 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /api/v1/superuser/users/ HTTP/2.0" 401 299 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.019 39 0.019) gunicorn-web stdout | 2025-11-04 09:32:30,593 [286] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:30 +0000] "GET /api/v1/superuser/users/ HTTP/1.0" 401 299 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" gunicorn-web stdout | 2025-11-04 09:32:30,596 [285] [DEBUG] [auth.permissions] Organization team added permission: _NamespaceWideNeed(type='organization', namespace='build_test', role='admin') gunicorn-web stdout | 2025-11-04 09:32:30,596 [285] [DEBUG] [auth.permissions] Organization team added repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='build_test', role='admin') gunicorn-web stdout | 2025-11-04 09:32:30,596 [285] [DEBUG] [auth.permissions] Team added permission: _TeamTypeNeed(type='orgteam', orgname='build_test', teamname='owners', role='admin') gunicorn-web stdout | 2025-11-04 09:32:30,596 [285] [DEBUG] [auth.permissions] Adding superuser to user: whuquay gunicorn-web stdout | 2025-11-04 09:32:30,596 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."organization_id", "t1"."role_id", "t1"."description" FROM "team" AS "t1" INNER JOIN "teamrole" AS "t2" ON ("t1"."role_id" = "t2"."id") WHERE ("t1"."organization_id" = %s)', [2]) gunicorn-web stdout | 2025-11-04 09:32:30,600 [285] [DEBUG] [peewee] ('SELECT "t1"."team_id", Count("t1"."id") FROM "repositorypermission" AS "t1" WHERE ("t1"."team_id" IN (%s)) GROUP BY "t1"."team_id"', [1]) gunicorn-web stdout | 2025-11-04 09:32:30,603 [285] [DEBUG] [peewee] ('SELECT "t1"."team_id", Count("t1"."id") FROM "teammember" AS "t1" WHERE ("t1"."team_id" IN (%s)) GROUP BY "t1"."team_id"', [1]) gunicorn-web stdout | 2025-11-04 09:32:30,606 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,606 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,606 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,606 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,606 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-web stdout | 2025-11-04 09:32:30,610 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,613 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-web stdout | 2025-11-04 09:32:30,616 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,619 [285] [DEBUG] [app] Ending request: urn:request:0569ae39-331f-4102-9a4a-dcd013c74e61 (/api/v1/organization/build_test) {'endpoint': 'api.organization', 'request_id': 'urn:request:0569ae39-331f-4102-9a4a-dcd013c74e61', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/organization/build_test', 'path': '/api/v1/organization/build_test', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:30,619 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:30,620 [285] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:30 +0000] "GET /api/v1/organization/build_test HTTP/1.0" 200 813 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /api/v1/organization/build_test HTTP/2.0" 200 813 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.046 973 0.046) gunicorn-web stdout | 2025-11-04 09:32:30,696 [285] [DEBUG] [app] Starting request: urn:request:20178b45-9cd6-4a79-9b8d-84a74230bf81 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:30,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:30,700 [292] [DEBUG] [app] Starting request: urn:request:e73e0a09-a44f-439c-b016-ca814319cc9f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:30,701 [292] [DEBUG] [app] Ending request: urn:request:e73e0a09-a44f-439c-b016-ca814319cc9f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e73e0a09-a44f-439c-b016-ca814319cc9f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:30,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:32:30,701 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:30,705 [288] [DEBUG] [app] Starting request: urn:request:e6e4776b-71f4-4517-86da-feb058c184d3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:30,705 [288] [DEBUG] [app] Ending request: urn:request:e6e4776b-71f4-4517-86da-feb058c184d3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e6e4776b-71f4-4517-86da-feb058c184d3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:32:30,705 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:32:30,705 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:30,713 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:30,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:30,726 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:30,729 [285] [DEBUG] [app] Ending request: urn:request:20178b45-9cd6-4a79-9b8d-84a74230bf81 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:20178b45-9cd6-4a79-9b8d-84a74230bf81', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:30,729 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) gunicorn-web stdout | 2025-11-04 09:32:30,783 [288] [DEBUG] [app] Starting request: urn:request:a3b4b6b7-9e59-465c-b4aa-825afecbddff (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:30,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:30,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:30,787 [292] [DEBUG] [app] Starting request: urn:request:d11ea8b2-9d50-474c-b210-a47af035422d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:30,788 [292] [DEBUG] [app] Ending request: urn:request:d11ea8b2-9d50-474c-b210-a47af035422d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d11ea8b2-9d50-474c-b210-a47af035422d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:32:30,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:30,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:30,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:30,792 [285] [DEBUG] [app] Starting request: urn:request:3155eccd-9264-42d2-83ff-cc7ee62a43cf (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:30,792 [285] [DEBUG] [app] Ending request: urn:request:3155eccd-9264-42d2-83ff-cc7ee62a43cf (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3155eccd-9264-42d2-83ff-cc7ee62a43cf', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:32:30,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:32:30,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:30,792 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:30,792 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:30,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:30,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:30,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:30,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:30,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:30,816 [288] [DEBUG] [app] Ending request: urn:request:a3b4b6b7-9e59-465c-b4aa-825afecbddff (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a3b4b6b7-9e59-465c-b4aa-825afecbddff', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:30,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:30,816 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) gunicorn-web stdout | 2025-11-04 09:32:30,817 [285] [DEBUG] [app] Starting request: urn:request:b64d4dae-1ed8-452e-abf2-733169f40e6b (/api/v1/superuser/organizations/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:30,818 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,818 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,818 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,828 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:30,828 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:30,828 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:30,828 [285] [DEBUG] [endpoints.api] Checking fresh login for user whuquay: Last login at 2025-11-04 07:15:43+00:00 gunicorn-web stdout | 2025-11-04 09:32:30,829 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['whuquay', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:30,832 [285] [DEBUG] [app] Ending request: urn:request:b64d4dae-1ed8-452e-abf2-733169f40e6b (/api/v1/superuser/organizations/) {'endpoint': 'api.superuserorganizationlist', 'request_id': 'urn:request:b64d4dae-1ed8-452e-abf2-733169f40e6b', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/superuser/organizations/', 'path': '/api/v1/superuser/organizations/', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:30,833 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:30 +0000] "GET /api/v1/superuser/organizations/ HTTP/2.0" 401 299 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.016 974 0.016) gunicorn-web stdout | 2025-11-04 09:32:30,833 [285] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:30 +0000] "GET /api/v1/superuser/organizations/ HTTP/1.0" 401 299 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" manifestsubjectbackfillworker stdout | 2025-11-04 09:32:31,515 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:32:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:32:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:41.530064+00:00 (in 9.999550 seconds) notificationworker stdout | 2025-11-04 09:32:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:41 GMT)" (scheduled at 2025-11-04 09:32:31.530064+00:00) notificationworker stdout | 2025-11-04 09:32:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:32:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 31, 530784), True, datetime.datetime(2025, 11, 4, 9, 32, 31, 530784), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:32:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:32:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:32:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:32:32,312 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:32:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:32:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:08.435432+00:00 (in 35.999221 seconds) namespacegcworker stdout | 2025-11-04 09:32:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:32 GMT)" (scheduled at 2025-11-04 09:32:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:32:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:32:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:32 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:32:32,500 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:32:32,509 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:32:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:32:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:48.625192+00:00 (in 15.001797 seconds) exportactionlogsworker stdout | 2025-11-04 09:32:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:33 GMT)" (scheduled at 2025-11-04 09:32:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:32:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:32:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 33, 623706), True, datetime.datetime(2025, 11, 4, 9, 32, 33, 623706), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:32:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:32:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:32:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:32:33,799 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:32:33,800 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:32:33,891 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:32:34,672 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:32:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:32:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:38.437104+00:00 (in 3.000915 seconds) gcworker stdout | 2025-11-04 09:32:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:05 GMT)" (scheduled at 2025-11-04 09:32:35.435809+00:00) gcworker stdout | 2025-11-04 09:32:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039155436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:32:35,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:32:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:32:35,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:32:35,989 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:32:36,358 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:36 +0000] "GET /vendor.css.map HTTP/2.0" 200 457255 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.014 699 -) securityworker stdout | 2025-11-04 09:32:36,977 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:37 +0000] "GET /main.css.map HTTP/2.0" 200 8968 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.000 697 -) proxycacheblobworker stdout | 2025-11-04 09:32:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:32:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:40.124445+00:00 (in 3.001153 seconds) proxycacheblobworker stdout | 2025-11-04 09:32:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:47 GMT)" (scheduled at 2025-11-04 09:32:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:32:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:32:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 37, 123594), True, datetime.datetime(2025, 11, 4, 9, 32, 37, 123594), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:32:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:32:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:32:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:32:37,401 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:32:37,595 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:32:37,609 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:32:37,609 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:32:37,619 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:32:38,413 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:38,414 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:38,429 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:32:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:32:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:05.435809+00:00 (in 26.998342 seconds) gcworker stdout | 2025-11-04 09:32:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:08 GMT)" (scheduled at 2025-11-04 09:32:38.437104+00:00) gcworker stdout | 2025-11-04 09:32:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:32:38,443 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:32:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248698448, None, 1, 0]) gcworker stdout | 2025-11-04 09:32:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:32:38,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:32:38,466 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:38,482 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:38,513 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:38,663 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:32:38,667 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:32:38,823 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:32:38,824 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:32:38,814 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:32:39,528 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:32:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:32:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:01.922827+00:00 (in 21.997752 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:32:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:39 GMT)" (scheduled at 2025-11-04 09:32:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:32:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:32:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:32:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:32:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:47.122849+00:00 (in 6.997960 seconds) proxycacheblobworker stdout | 2025-11-04 09:32:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:40 GMT)" (scheduled at 2025-11-04 09:32:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:32:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:32:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:32:41,296 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:32:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:32:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:51.530064+00:00 (in 9.999518 seconds) notificationworker stdout | 2025-11-04 09:32:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:51 GMT)" (scheduled at 2025-11-04 09:32:41.530064+00:00) notificationworker stdout | 2025-11-04 09:32:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:32:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 41, 530862), True, datetime.datetime(2025, 11, 4, 9, 32, 41, 530862), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:32:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:32:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:32:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:32:42,747 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:32:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:32:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:57.523915+00:00 (in 14.090594 seconds) repositorygcworker stdout | 2025-11-04 09:32:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:43 GMT)" (scheduled at 2025-11-04 09:32:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:32:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:32:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 43, 433622), True, datetime.datetime(2025, 11, 4, 9, 32, 43, 433622), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:32:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:32:43,445 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:32:43,445 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:32:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:32:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:59.351216+00:00 (in 14.999453 seconds) quotatotalworker stdout | 2025-11-04 09:32:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:59 GMT)" (scheduled at 2025-11-04 09:32:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:32:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:32:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:32:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:32:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:32:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:32:45,253 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:32:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:32:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:47.479080+00:00 (in 1.998902 seconds) securityworker stdout | 2025-11-04 09:32:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:15 GMT)" (scheduled at 2025-11-04 09:32:45.479769+00:00) securityworker stdout | 2025-11-04 09:32:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:32:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:32:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:32:45,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:45,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 27, 45, 483795), 1, 2]) securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 27, 45, 483795), 1, 2]) securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:32:45,696 [285] [DEBUG] [app] Starting request: urn:request:33b6b764-2c65-4057-9b96-2fbd1eb2dc8c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:45,701 [299] [DEBUG] [app] Starting request: urn:request:e90fc80b-e039-4f3b-b6c3-8bf4de946a93 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:45,701 [299] [DEBUG] [app] Ending request: urn:request:e90fc80b-e039-4f3b-b6c3-8bf4de946a93 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e90fc80b-e039-4f3b-b6c3-8bf4de946a93', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:45,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:32:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:45,706 [288] [DEBUG] [app] Starting request: urn:request:4e9e0f33-b196-421e-9430-4079c85d43b5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:45,706 [288] [DEBUG] [app] Ending request: urn:request:4e9e0f33-b196-421e-9430-4079c85d43b5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4e9e0f33-b196-421e-9430-4079c85d43b5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:32:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:32:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:45,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:45,730 [285] [DEBUG] [app] Ending request: urn:request:33b6b764-2c65-4057-9b96-2fbd1eb2dc8c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:33b6b764-2c65-4057-9b96-2fbd1eb2dc8c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:45,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:32:45,784 [286] [DEBUG] [app] Starting request: urn:request:06b3f7c4-a6f7-4515-98c2-9b10f3801ea6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:32:45,785 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:45,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:32:45,788 [292] [DEBUG] [app] Starting request: urn:request:57b2ae50-5176-4d9d-ab51-3b5e3637086b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:32:45,788 [292] [DEBUG] [app] Ending request: urn:request:57b2ae50-5176-4d9d-ab51-3b5e3637086b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:57b2ae50-5176-4d9d-ab51-3b5e3637086b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:32:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:32:45,788 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:32:45,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:32:45,791 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:32:45,792 [288] [DEBUG] [app] Starting request: urn:request:bd498571-dc82-4ade-84d1-b4b44d75f65c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:32:45,792 [288] [DEBUG] [app] Ending request: urn:request:bd498571-dc82-4ade-84d1-b4b44d75f65c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bd498571-dc82-4ade-84d1-b4b44d75f65c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:32:45,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:32:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:32:45,793 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:32:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:32:45,793 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:32:45,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:32:45,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:32:45,800 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:32:45,800 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:32:45,810 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:32:45,813 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:32:45,816 [286] [DEBUG] [app] Ending request: urn:request:06b3f7c4-a6f7-4515-98c2-9b10f3801ea6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:06b3f7c4-a6f7-4515-98c2-9b10f3801ea6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:32:45,816 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:32:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) gunicorn-web stdout | 2025-11-04 09:32:45,817 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:32:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" proxycacheblobworker stdout | 2025-11-04 09:32:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:32:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:32:57.122849+00:00 (in 9.999571 seconds) proxycacheblobworker stdout | 2025-11-04 09:32:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:57 GMT)" (scheduled at 2025-11-04 09:32:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:32:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:32:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 47, 123582), True, datetime.datetime(2025, 11, 4, 9, 32, 47, 123582), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:32:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:32:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:32:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:32:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:32:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:32:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:15.479769+00:00 (in 28.000240 seconds) securityworker stdout | 2025-11-04 09:32:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:17 GMT)" (scheduled at 2025-11-04 09:32:47.479080+00:00) securityworker stdout | 2025-11-04 09:32:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:32:47,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:32:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:32:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:32:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 27, 47, 483058), 1, 2]) securityworker stdout | 2025-11-04 09:32:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:32:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:32:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:32:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 27, 47, 483058), 1, 2]) securityworker stdout | 2025-11-04 09:32:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:32:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:32:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:32:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:32:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:32:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:32:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:32:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:17 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:32:47,520 [288] [DEBUG] [app] Starting request: urn:request:7021f9bf-8855-4687-919f-9759ac2dcc85 (/api/v1/filedrop/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:47,520 [288] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:47,520 [288] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:47,521 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:47,532 [288] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:47,532 [288] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:47,532 [288] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:47,532 [288] [DEBUG] [data.userfiles] Requested upload url with content type: application/octet-stream gunicorn-web stdout | 2025-11-04 09:32:47,532 [288] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,532 [288] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,532 [288] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.PutObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event choose-signer.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.hooks] Event before-sign.s3.PutObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | PUT gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093247Z&X-Amz-Expires=300&X-Amz-SignedHeaders=content-type%3Bhost gunicorn-web stdout | content-type:application/octet-stream gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | content-type;host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093247Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | c28b2de9364ff0b286fb2a866eb8af0901c9fa738aa31fd947649f27dc474195 gunicorn-web stdout | 2025-11-04 09:32:47,533 [288] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | c2e3daab5f6474f1b8614145bc363c5311ccaf0f640d0419ff8c896c28e4a84d gunicorn-web stdout | 2025-11-04 09:32:47,534 [288] [DEBUG] [app] Ending request: urn:request:7021f9bf-8855-4687-919f-9759ac2dcc85 (/api/v1/filedrop/) {'endpoint': 'api.filedropresource', 'request_id': 'urn:request:7021f9bf-8855-4687-919f-9759ac2dcc85', 'remote_addr': '10.129.2.10', 'http_method': 'POST', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/filedrop/', 'path': '/api/v1/filedrop/', 'parameters': {}, 'json_body': {'mimeType': 'application/octet-stream'}, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:47,534 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:47,534 [288] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:47 +0000] "POST /api/v1/filedrop/ HTTP/1.0" 200 441 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:47 +0000] "POST /api/v1/filedrop/ HTTP/2.0" 200 441 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.015 1093 0.016) builder stdout | 2025-11-04 09:32:47,619 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:32:47,632 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:32:47,632 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:32:47,642 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:32:48,189 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:32:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:32:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:33.622985+00:00 (in 44.997344 seconds) exportactionlogsworker stdout | 2025-11-04 09:32:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:48 GMT)" (scheduled at 2025-11-04 09:32:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:32:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:32:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:32:48,862 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:50,047 [287] [DEBUG] [app] Starting request: urn:request:ceddd0bc-78b9-424b-9e64-0ed20b7174e5 (/api/v1/repository/build_test/upload_file_firefox/build/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:50,047 [287] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,047 [287] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,048 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,060 [287] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,061 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:50,066 [287] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,067 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:50,072 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,076 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,080 [287] [DEBUG] [endpoints.api.build] User requested repository initialization. gunicorn-web stdout | 2025-11-04 09:32:50,080 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorybuild" AS "t3" ON ("t3"."repository_id" = "t1"."id") WHERE ("t3"."resource_key" = %s) LIMIT %s OFFSET %s', ['3b4ed7ae-c4a4-44f4-b856-4499356fd36b', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,085 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,088 [287] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-web stdout | 2025-11-04 09:32:50,089 [287] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fa67d7f5b20> gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'whuaws', 'Key': 'quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b'} gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:32:50,090 [287] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | HEAD gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-web stdout | x-amz-date:20251104T093250Z gunicorn-web stdout | host;x-amz-content-sha256;x-amz-date gunicorn-web stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093250Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | cca721a2d88f34eee4b8c3433b0ab25ab2c8ffb8f0c20eddd9e621de2d53cfc6 gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | 8f7f0a6f884df0991bc3cd7614b3a5a772d624ddae8e20fdcd5df43d48206ebd gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093250Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8f7f0a6f884df0991bc3cd7614b3a5a772d624ddae8e20fdcd5df43d48206ebd', 'amz-sdk-invocation-id': b'd1dde3ab-b066-44da-830d-dbefd610facb', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-web stdout | 2025-11-04 09:32:50,091 [287] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-web stdout | 2025-11-04 09:32:50,092 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: s3.us-east-1.amazonaws.com repositorygcworker stdout | 2025-11-04 09:32:50,171 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:50,230 [287] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "HEAD /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b HTTP/1.1" 200 0 gunicorn-web stdout | 2025-11-04 09:32:50,230 [287] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'QxapL09cB5scvw4EtRUxAiyfSvizJd8t9rjPiZ7Alv11ICrtLICUd3emcoQf+19fQyhmXWOWpi3HY/eWrJqBEA==', 'x-amz-request-id': '8BEPA1ATE7GJ6ZPQ', 'Date': 'Tue, 04 Nov 2025 09:32:51 GMT', 'Last-Modified': 'Tue, 04 Nov 2025 09:32:50 GMT', 'ETag': '"0f73826affbfdba58fdd744db7afbe65"', 'x-amz-server-side-encryption': 'AES256', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/octet-stream', 'Content-Length': '514', 'Server': 'AmazonS3'} gunicorn-web stdout | 2025-11-04 09:32:50,230 [287] [DEBUG] [botocore.parsers] Response body: gunicorn-web stdout | b'' gunicorn-web stdout | 2025-11-04 09:32:50,231 [287] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,231 [287] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-web stdout | 2025-11-04 09:32:50,231 [287] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,231 [287] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,231 [287] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': '8BEPA1ATE7GJ6ZPQ', 'HostId': 'QxapL09cB5scvw4EtRUxAiyfSvizJd8t9rjPiZ7Alv11ICrtLICUd3emcoQf+19fQyhmXWOWpi3HY/eWrJqBEA==', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'QxapL09cB5scvw4EtRUxAiyfSvizJd8t9rjPiZ7Alv11ICrtLICUd3emcoQf+19fQyhmXWOWpi3HY/eWrJqBEA==', 'x-amz-request-id': '8BEPA1ATE7GJ6ZPQ', 'date': 'Tue, 04 Nov 2025 09:32:51 GMT', 'last-modified': 'Tue, 04 Nov 2025 09:32:50 GMT', 'etag': '"0f73826affbfdba58fdd744db7afbe65"', 'x-amz-server-side-encryption': 'AES256', 'accept-ranges': 'bytes', 'content-type': 'application/octet-stream', 'content-length': '514', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 9, 32, 50, tzinfo=tzutc()), 'ContentLength': 514, 'ETag': '"0f73826affbfdba58fdd744db7afbe65"', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256', 'Metadata': {}} gunicorn-web stdout | 2025-11-04 09:32:50,232 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,235 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "role" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['write', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,239 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "accesstokenkind" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['build-worker', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,243 [287] [DEBUG] [peewee] ('INSERT INTO "accesstoken" ("friendly_name", "token_name", "token_code", "repository_id", "created", "role_id", "temporary", "kind_id") VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING "accesstoken"."id"', ['Repository Build Token', 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 'v0$$7HXn03Xr8K2f3BLtvo4pDzufWC8bkbmmHb18tNhUSdbi8YMc7sK67CHL+tg12Nu3M9LmZInnRRUYf4J6OQ==', 2, datetime.datetime(2025, 11, 4, 9, 32, 50, 242693), 2, True, 1]) gunicorn-web stdout | 2025-11-04 09:32:50,248 [287] [DEBUG] [endpoints.building] Creating build 0f73826 with repo quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/build_test/upload_file_firefox tags ['latest'] gunicorn-web stdout | 2025-11-04 09:32:50,248 [287] [DEBUG] [peewee] ('INSERT INTO "repositorybuild" ("uuid", "repository_id", "access_token_id", "resource_key", "job_config", "phase", "started", "display_name", "trigger_id", "pull_robot_id", "logs_archived") VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING "repositorybuild"."id"', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 2, 3, '3b4ed7ae-c4a4-44f4-b856-4499356fd36b', '{"docker_tags": ["latest"], "registry": "quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com", "build_subdir": "/Dockerfile", "context": "/", "trigger_metadata": {}, "is_manual": true, "manual_user": "whuquay", "archive_url": null}', 'waiting', datetime.datetime(2025, 11, 4, 9, 32, 50, 248211), '0f73826', None, None, False]) gunicorn-web stdout | 2025-11-04 09:32:50,251 [287] [DEBUG] [peewee] ('INSERT INTO "queueitem" ("queue_name", "body", "available_after", "available", "retries_remaining", "state_id") VALUES (%s, %s, %s, %s, %s, %s) RETURNING "queueitem"."id"', ['dockerfilebuild/build_test/upload_file_firefox/', '{"build_uuid": "4708783e-59eb-4e42-b7a1-1923a4b4caff", "pull_credentials": null}', datetime.datetime(2025, 11, 4, 9, 32, 50, 251062), True, 3, 'c72bfb11-ee54-4174-9068-c866876e4aa3']) gunicorn-web stdout | 2025-11-04 09:32:50,252 [287] [DEBUG] [peewee] ('UPDATE "repositorybuild" SET "uuid" = %s, "repository_id" = %s, "access_token_id" = %s, "resource_key" = %s, "job_config" = %s, "phase" = %s, "started" = %s, "display_name" = %s, "trigger_id" = %s, "pull_robot_id" = %s, "logs_archived" = %s, "queue_id" = %s WHERE ("repositorybuild"."id" = %s)', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 2, 3, '3b4ed7ae-c4a4-44f4-b856-4499356fd36b', '{"docker_tags": ["latest"], "registry": "quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com", "build_subdir": "/Dockerfile", "context": "/", "trigger_metadata": {}, "is_manual": true, "manual_user": "whuquay", "archive_url": null}', 'waiting', datetime.datetime(2025, 11, 4, 9, 32, 50, 248211), '0f73826', None, None, False, '3', 3]) gunicorn-web stdout | 2025-11-04 09:32:50,256 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,260 [287] [DEBUG] [peewee] ('INSERT INTO "logentry3" ("kind_id", "account_id", "performer_id", "repository_id", "datetime", "ip", "metadata_json") VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING "logentry3"."id"', [9, 2, None, 2, datetime.datetime(2025, 11, 4, 9, 32, 50, 256428), '10.129.2.10', '{"build_id": "4708783e-59eb-4e42-b7a1-1923a4b4caff", "docker_tags": ["latest"], "repo": "upload_file_firefox", "namespace": "build_test", "is_manual": true, "manual_user": "whuquay"}']) gunicorn-web stdout | 2025-11-04 09:32:50,265 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."event_id", "t1"."method_id", "t1"."title", "t1"."config_json", "t1"."event_config_json", "t1"."number_of_failures", "t1"."last_ran_ms", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed" FROM "repositorynotification" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") INNER JOIN "externalnotificationevent" AS "t4" ON ("t1"."event_id" = "t4"."id") WHERE ((("t3"."username" = %s) AND ("t2"."name" = %s)) AND ("t4"."name" = %s))', ['build_test', 'upload_file_firefox', 'build_queued']) gunicorn-web stdout | 2025-11-04 09:32:50,268 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,268 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,268 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093250Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093250Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | 5aff407ad2bbe543388830e05fc24666e3666dc7779fbf3fb497465613ecfe3c gunicorn-web stdout | 2025-11-04 09:32:50,269 [287] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | a304a0013a46140dd962488712bb4d1984e169a13ddfedb1f11cd8ff5cff7002 gunicorn-web stdout | 2025-11-04 09:32:50,270 [287] [DEBUG] [app] Ending request: urn:request:ceddd0bc-78b9-424b-9e64-0ed20b7174e5 (/api/v1/repository/build_test/upload_file_firefox/build/) {'endpoint': 'api.repositorybuildlist', 'request_id': 'urn:request:ceddd0bc-78b9-424b-9e64-0ed20b7174e5', 'remote_addr': '10.129.2.10', 'http_method': 'POST', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/', 'parameters': {}, 'json_body': {'file_id': '3b4ed7ae-c4a4-44f4-b856-4499356fd36b'}, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:50,270 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:50,271 [287] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:50 +0000] "POST /api/v1/repository/build_test/upload_file_firefox/build/ HTTP/1.0" 201 888 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:50 +0000] "POST /api/v1/repository/build_test/upload_file_firefox/build/ HTTP/2.0" 201 888 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.227 464 0.224) gunicorn-web stdout | 2025-11-04 09:32:50,584 [285] [DEBUG] [app] Starting request: urn:request:a9daf571-217b-45ad-aba2-dd098fb625fe (/api/v1/repository/build_test/upload_file_firefox/build/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:50,584 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,584 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,585 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,596 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:50,597 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:50,597 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,597 [285] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:32:50,597 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,597 [285] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,597 [285] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,597 [285] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,598 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:50,602 [285] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:32:50,603 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:50,608 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:50,613 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE (("t4"."name" = %s) AND ("t5"."username" = %s)) ORDER BY "t1"."started" DESC LIMIT %s', ['upload_file_firefox', 'build_test', 10]) gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,619 [285] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093250Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093250Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | 5aff407ad2bbe543388830e05fc24666e3666dc7779fbf3fb497465613ecfe3c gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | a304a0013a46140dd962488712bb4d1984e169a13ddfedb1f11cd8ff5cff7002 gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,620 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/7da4d504-2eb2-4d26-9424-a797cd4a5b18 gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093250Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093250Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | d4159c5d7b39e954f6f162a9ee87465cb266768a608c1a0b445a35a63771e0cc gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | 571cfc684c18cc673478746828420737420b179b537c293f8cc514dd83cb8af3 gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,621 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/6647c276-7aad-46a3-a070-72e4f1b52efe gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093250Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093250Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | c0bf1c4c2051675d5edee0a4b5afa199b4582fdacd8a3bff8fae6958bf87eaaf gunicorn-web stdout | 2025-11-04 09:32:50,622 [285] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | 0d6ce05f1aab451469a3b20fa54f7ae6c2b53c4f45c142f339f77875e63b4a54 gunicorn-web stdout | 2025-11-04 09:32:50,623 [285] [DEBUG] [app] Ending request: urn:request:a9daf571-217b-45ad-aba2-dd098fb625fe (/api/v1/repository/build_test/upload_file_firefox/build/) {'endpoint': 'api.repositorybuildlist', 'request_id': 'urn:request:a9daf571-217b-45ad-aba2-dd098fb625fe', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/?limit=10', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/', 'parameters': {'limit': '10'}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:50,623 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:50,623 [285] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:50 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/?limit=10 HTTP/1.0" 200 2683 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:50 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/?limit=10 HTTP/2.0" 200 2683 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox?tab=builds" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.040 414 0.041) autopruneworker stdout | 2025-11-04 09:32:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:32:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:21.226294+00:00 (in 29.999535 seconds) autopruneworker stdout | 2025-11-04 09:32:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:21 GMT)" (scheduled at 2025-11-04 09:32:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:32:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248711234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:32:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:32:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:32:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:32:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:32:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:01.530064+00:00 (in 9.999599 seconds) notificationworker stdout | 2025-11-04 09:32:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:01 GMT)" (scheduled at 2025-11-04 09:32:51.530064+00:00) notificationworker stdout | 2025-11-04 09:32:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:32:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 51, 530740), True, datetime.datetime(2025, 11, 4, 9, 32, 51, 530740), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:32:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:32:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:32:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:32:52,440 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:32:54,555 [285] [DEBUG] [app] Starting request: urn:request:5dc8a063-0b43-4b6d-9655-ab4bcece943f (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:54,555 [286] [DEBUG] [app] Starting request: urn:request:be1f6237-e129-45eb-9ca5-a4f680036d20 (/api/v1/repository/build_test/upload_file_firefox) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:54,556 [288] [DEBUG] [app] Starting request: urn:request:aece72f4-d67f-494e-8458-795d7ec71b33 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:32:54,556 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,556 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,556 [288] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,556 [286] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,556 [288] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,556 [286] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,556 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,556 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,557 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:32:54,568 [286] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:54,568 [288] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,568 [288] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:54,568 [286] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:32:54,568 [288] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,568 [285] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,568 [286] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,568 [288] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:32:54,568 [288] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,568 [288] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,569 [288] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,569 [288] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,569 [286] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:32:54,569 [286] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,569 [286] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,569 [286] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,569 [286] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,569 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:54,570 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:54,570 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:54,574 [288] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,575 [285] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,575 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:54,575 [286] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:32:54,576 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:54,576 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:32:54,580 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,581 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,581 [286] [DEBUG] [endpoints.api.repository] Get repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:32:54,582 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,584 [288] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,585 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."user_id", "t1"."repository_id", "t1"."created" FROM "star" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."user_id" = %s)) LIMIT %s OFFSET %s', [2, 1, 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,585 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,586 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) gunicorn-web stdout | 2025-11-04 09:32:54,589 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."count", "t1"."date" FROM "repositoryactioncount" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."date" >= %s))', [2, datetime.date(2025, 8, 4)]) gunicorn-web stdout | 2025-11-04 09:32:54,591 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,591 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,591 [288] [DEBUG] [app] Ending request: urn:request:aece72f4-d67f-494e-8458-795d7ec71b33 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'endpoint': 'api.repositorybuildlogs', 'request_id': 'urn:request:aece72f4-d67f-494e-8458-795d7ec71b33', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=0', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs', 'parameters': {'start': '0'}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:32:54,592 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:32:54,592 [286] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,592 [288] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:54 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=0 HTTP/1.0" 200 37 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:32:54,592 [286] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:54 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=0 HTTP/2.0" 200 37 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.038 97 0.038) gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:32:54,592 [285] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093254Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:32:54,593 [285] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093254Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | 8e518f5fb2f8769a9e791aaf62f58793bacecf3fa80d355ce184e28633435428 gunicorn-web stdout | 2025-11-04 09:32:54,593 [285] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | 6f598378f1fc6ba41dfc267394a520316f5bb0d03cbd9d7e4c97903f060f1162 gunicorn-web stdout | 2025-11-04 09:32:54,593 [285] [DEBUG] [app] Ending request: urn:request:5dc8a063-0b43-4b6d-9655-ab4bcece943f (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'endpoint': 'api.repositorybuildresource', 'request_id': 'urn:request:5dc8a063-0b43-4b6d-9655-ab4bcece943f', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:54,593 [286] [DEBUG] [app] Ending request: urn:request:be1f6237-e129-45eb-9ca5-a4f680036d20 (/api/v1/repository/build_test/upload_file_firefox) {'endpoint': 'api.repository', 'request_id': 'urn:request:be1f6237-e129-45eb-9ca5-a4f680036d20', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox?includeStats=true&includeTags=false', 'path': '/api/v1/repository/build_test/upload_file_firefox', 'parameters': {'includeStats': 'true', 'includeTags': 'false'}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:32:54,593 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:32:54,594 [285] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:54 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/1.0" 200 888 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" gunicorn-web stdout | 2025-11-04 09:32:54,594 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:54 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/2.0" 200 888 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.040 216 0.039) gunicorn-web stdout | 2025-11-04 09:32:54,594 [286] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:32:54 +0000] "GET /api/v1/repository/build_test/upload_file_firefox?includeStats=true&includeTags=false HTTP/1.0" 200 3602 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:54 +0000] "GET /api/v1/repository/build_test/upload_file_firefox?includeStats=true&includeTags=false HTTP/2.0" 200 3602 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.040 83 0.040) notificationworker stdout | 2025-11-04 09:32:55,298 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:32:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:32:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:07.122849+00:00 (in 9.999573 seconds) proxycacheblobworker stdout | 2025-11-04 09:32:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:07 GMT)" (scheduled at 2025-11-04 09:32:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:32:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:32:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 32, 57, 123560), True, datetime.datetime(2025, 11, 4, 9, 32, 57, 123560), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:32:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:32:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:09.133645+00:00 (in 12.001516 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:32:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:57 GMT)" (scheduled at 2025-11-04 09:32:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:32:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:32:57,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:32:57,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:32:57,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:32:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:32:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:32:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:32:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:32:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:27.222846+00:00 (in 29.999566 seconds) buildlogsarchiver stdout | 2025-11-04 09:32:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:27 GMT)" (scheduled at 2025-11-04 09:32:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:32:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 32, 57, 223484), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:32:57,239 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:32:57,242 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:32:57,243 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:32:57,249 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:32:57,250 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:27 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:32:57,521 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:32:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:32:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:43.432867+00:00 (in 45.908553 seconds) repositorygcworker stdout | 2025-11-04 09:32:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:57 GMT)" (scheduled at 2025-11-04 09:32:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:32:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:32:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:33:57 GMT)" executed successfully builder stdout | 2025-11-04 09:32:57,642 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:32:57,655 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:32:57,655 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:32:57,665 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:32:58,242 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:32:58,895 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:32:59,277 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:32:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:32:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:14.351216+00:00 (in 14.999579 seconds) quotatotalworker stdout | 2025-11-04 09:32:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:14 GMT)" (scheduled at 2025-11-04 09:32:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:32:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:32:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:32:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:32:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:32:59,677 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:59 +0000] "POST /buildman_pb.BuildManager/Ping HTTP/2.0" 200 11 "-" "grpc-go/1.71.0" (0.001 156 0.001) builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler builder stdout | 2025-11-04 09:32:59,701 [66] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler builder stdout | 2025-11-04 09:32:59,702 [66] [DEBUG] [botocore.auth] Calculating signature using v4 auth. builder stdout | 2025-11-04 09:32:59,702 [66] [DEBUG] [botocore.auth] CanonicalRequest: builder stdout | GET builder stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b builder stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093259Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host builder stdout | host:s3.us-east-1.amazonaws.com builder stdout | host builder stdout | UNSIGNED-PAYLOAD builder stdout | 2025-11-04 09:32:59,702 [66] [DEBUG] [botocore.auth] StringToSign: builder stdout | AWS4-HMAC-SHA256 builder stdout | 20251104T093259Z builder stdout | 20251104/us-east-1/s3/aws4_request builder stdout | 78b23887c2ad93b72f98f1e6dbaa592acdaf9ec2ad4e9333062008f2a9c7c83a builder stdout | 2025-11-04 09:32:59,702 [66] [DEBUG] [botocore.auth] Signature: builder stdout | 82c4d0f73817ae9581865e1fc0d803292eeeabe3744639cce530dbfabd13185d builder stdout | 2025-11-04 09:32:59,754 [66] [DEBUG] [buildman.manager.ephemeral] Started build job buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff with arguments {'build_package': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093259Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host&X-Amz-Signature=82c4d0f73817ae9581865e1fc0d803292eeeabe3744639cce530dbfabd13185d', 'context': '/', 'dockerfile_path': 'Dockerfile', 'repository': 'build_test/upload_file_firefox', 'registry': 'quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com', 'pull_token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'push_token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'tag_names': ['latest'], 'base_image': {}} nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:59 +0000] "POST /buildman_pb.BuildManager/RegisterBuildJob HTTP/2.0" 200 1617 "-" "grpc-go/1.71.0" (0.068 1032 0.069) builder stdout | 2025-11-04 09:32:59,768 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:32:59,772 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:32:59.758678\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:32:59,783 [66] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:59 +0000] "POST /buildman_pb.BuildManager/SetPhase HTTP/2.0" 200 9 "-" "grpc-go/1.71.0" (0.025 1025 0.026) builder stdout | 2025-11-04 09:32:59,853 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:32:59,871 [66] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:32:59 +0000] "POST /buildman_pb.BuildManager/SetPhase HTTP/2.0" 200 9 "-" "grpc-go/1.71.0" (0.032 1129 0.032) gunicorn-web stdout | 2025-11-04 09:32:59,944 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:00,129 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:00,142 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:00,234 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:00,696 [288] [DEBUG] [app] Starting request: urn:request:9d36e14b-fba2-411f-b549-2c14620c2173 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:00,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:00,701 [299] [DEBUG] [app] Starting request: urn:request:df59998d-16c0-4d99-8e6e-7db3707dff69 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:00,701 [299] [DEBUG] [app] Ending request: urn:request:df59998d-16c0-4d99-8e6e-7db3707dff69 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:df59998d-16c0-4d99-8e6e-7db3707dff69', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:33:00,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:33:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:00,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:00,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:00,705 [285] [DEBUG] [app] Starting request: urn:request:fdb0ae61-946c-43c5-9471-2eae94555fc5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:00,706 [285] [DEBUG] [app] Ending request: urn:request:fdb0ae61-946c-43c5-9471-2eae94555fc5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:fdb0ae61-946c-43c5-9471-2eae94555fc5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:33:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:33:00,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:00,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:00,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:00,730 [288] [DEBUG] [app] Ending request: urn:request:9d36e14b-fba2-411f-b549-2c14620c2173 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9d36e14b-fba2-411f-b549-2c14620c2173', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:00,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:33:00,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:33:00,784 [285] [DEBUG] [app] Starting request: urn:request:0cfbb63e-2b43-489f-b4f1-85fc763e4648 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:00,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:00,788 [299] [DEBUG] [app] Starting request: urn:request:23d88499-afc1-4058-b745-bf9b30aaa1f0 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:00,788 [299] [DEBUG] [app] Ending request: urn:request:23d88499-afc1-4058-b745-bf9b30aaa1f0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:23d88499-afc1-4058-b745-bf9b30aaa1f0', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:33:00,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:33:00,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:00,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:00,792 [287] [DEBUG] [app] Starting request: urn:request:00b4080b-649d-44f8-8889-be3ce9449946 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:00,792 [287] [DEBUG] [app] Ending request: urn:request:00b4080b-649d-44f8-8889-be3ce9449946 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:00b4080b-649d-44f8-8889-be3ce9449946', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:33:00,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:33:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:00,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:00,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:00,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:00,816 [285] [DEBUG] [app] Ending request: urn:request:0cfbb63e-2b43-489f-b4f1-85fc763e4648 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0cfbb63e-2b43-489f-b4f1-85fc763e4648', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:00,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.033) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:01,524 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:33:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:33:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:11.530064+00:00 (in 9.999588 seconds) notificationworker stdout | 2025-11-04 09:33:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:11 GMT)" (scheduled at 2025-11-04 09:33:01.530064+00:00) notificationworker stdout | 2025-11-04 09:33:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:33:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 1, 530772), True, datetime.datetime(2025, 11, 4, 9, 33, 1, 530772), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:33:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:33:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:33:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:11 GMT)" executed successfully builder stdout | 2025-11-04 09:33:01,782 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:01.776506\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} securityscanningnotificationworker stdout | 2025-11-04 09:33:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:33:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:39.924652+00:00 (in 38.001437 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:33:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:01 GMT)" (scheduled at 2025-11-04 09:33:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:33:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:33:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 1, 923452), True, datetime.datetime(2025, 11, 4, 9, 33, 1, 923452), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:33:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:33:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:33:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:33:02,321 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:02 +0000] "GET /vendor.bundle.js.map HTTP/2.0" 200 2502599 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.092 704 -) repositoryactioncounter stdout | 2025-11-04 09:33:02,517 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:33:02,510 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:03,793 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:03.787524\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} chunkcleanupworker stdout | 2025-11-04 09:33:03,808 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:33:03,808 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:33:03,912 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:04,683 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:04,882 [287] [DEBUG] [app] Starting request: urn:request:14144d0d-6dc8-4dd7-b575-53505b1621ce (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:33:04,882 [286] [DEBUG] [app] Starting request: urn:request:068cbf44-9809-4906-8408-13b35e348d4c (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:33:04,882 [287] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,882 [286] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,882 [287] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,882 [286] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,883 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:04,883 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:33:04,895 [286] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,895 [287] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,896 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:04,896 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:04,901 [286] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,901 [287] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:33:04,902 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:04,902 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:04,907 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:04,907 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:04,911 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,912 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:04,912 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:04,918 [287] [DEBUG] [app] Ending request: urn:request:14144d0d-6dc8-4dd7-b575-53505b1621ce (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'endpoint': 'api.repositorybuildlogs', 'request_id': 'urn:request:14144d0d-6dc8-4dd7-b575-53505b1621ce', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=0', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs', 'parameters': {'start': '0'}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:33:04,918 [286] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,918 [286] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,918 [286] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:04,918 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:04,918 [286] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:33:04,919 [287] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:04 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=0 HTTP/1.0" 200 508 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:04 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=0 HTTP/2.0" 200 508 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.038 388 0.038) gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093304Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093304Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | 8fd6764e3023f7ddf989cbac7ca17241e27ce0b653d24be4e440ca25d3d47513 gunicorn-web stdout | 2025-11-04 09:33:04,919 [286] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | da61e1c261598d8c04260b7c3b062cd2cac4e67ca71b33d8ed511fb390126623 gunicorn-web stdout | 2025-11-04 09:33:04,920 [286] [DEBUG] [app] Ending request: urn:request:068cbf44-9809-4906-8408-13b35e348d4c (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'endpoint': 'api.repositorybuildresource', 'request_id': 'urn:request:068cbf44-9809-4906-8408-13b35e348d4c', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:33:04,920 [286] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:04 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/2.0" 200 888 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.039 87 0.040) gunicorn-web stdout | 2025-11-04 09:33:04,920 [286] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:04 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/1.0" 200 888 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" gcworker stdout | 2025-11-04 09:33:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:33:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:08.437104+00:00 (in 3.000907 seconds) gcworker stdout | 2025-11-04 09:33:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:35 GMT)" (scheduled at 2025-11-04 09:33:05.435809+00:00) gcworker stdout | 2025-11-04 09:33:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039185436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:33:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:33:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:33:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:35 GMT)" executed successfully builder stdout | 2025-11-04 09:33:05,804 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:05.798578\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:33:05,998 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:33:06,367 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:33:06,986 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:33:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:33:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:17.122849+00:00 (in 9.999605 seconds) proxycacheblobworker stdout | 2025-11-04 09:33:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:17 GMT)" (scheduled at 2025-11-04 09:33:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:33:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:33:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 7, 123560), True, datetime.datetime(2025, 11, 4, 9, 33, 7, 123560), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:33:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:33:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:33:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:17 GMT)" executed successfully builder stdout | 2025-11-04 09:33:07,198 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:07,198 [66] [WARNING] [buildman.manager.ephemeral] Job buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff is already in the desired state/phase (pulling) nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:07 +0000] "POST /buildman_pb.BuildManager/SetPhase HTTP/2.0" 200 9 "-" "grpc-go/1.71.0" (0.023 998 0.023) builder stdout | 2025-11-04 09:33:07,225 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:07,237 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:07,237 [66] [DEBUG] [buildman.jobutil.buildjob] Determined cached tag latest for a084eb42a557707d65c2bf0fcd683648c7d9fcbbb56e596f221903944eeff7e7 nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:07 +0000] "POST /buildman_pb.BuildManager/DetermineCachedTag HTTP/2.0" 200 13 "-" "grpc-go/1.71.0" (0.029 1137 0.029) builder stdout | 2025-11-04 09:33:07,256 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:07,256 [66] [WARNING] [buildman.manager.ephemeral] Job buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff is already in the desired state/phase (pulling) nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:07 +0000] "POST /buildman_pb.BuildManager/SetPhase HTTP/2.0" 200 9 "-" "grpc-go/1.71.0" (0.016 998 0.017) gunicorn-registry stdout | 2025-11-04 09:33:07,277 [292] [DEBUG] [app] Starting request: urn:request:a0c51732-d8fa-440e-bca8-688e0ad3013f (/v2/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:07,277 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {} gunicorn-registry stdout | 2025-11-04 09:33:07,277 [292] [DEBUG] [auth.registry_jwt_auth] No auth header. gunicorn-registry stdout | 2025-11-04 09:33:07,277 [292] [DEBUG] [app] Ending request: urn:request:a0c51732-d8fa-440e-bca8-688e0ad3013f (/v2/) {'endpoint': 'v2.v2_support_enabled', 'request_id': 'urn:request:a0c51732-d8fa-440e-bca8-688e0ad3013f', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/', 'path': '/v2/', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:07,277 [292] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:07 +0000] "GET /v2/ HTTP/1.1" 401 4 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:07 +0000] "GET /v2/ HTTP/1.1" 401 4 "-" "containers/5.29.5 (github.com/containers/image)" (0.001 243 0.002) gunicorn-registry stdout | 2025-11-04 09:33:07,303 [292] [DEBUG] [app] Starting request: urn:request:477c49c6-627f-480b-9607-45bb14168352 (/v2/auth) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:07,303 [292] [DEBUG] [auth.basic] Attempt to process basic auth header gunicorn-registry stdout | 2025-11-04 09:33:07,304 [292] [DEBUG] [auth.credentials] Found credentials for access token gunicorn-registry stdout | 2025-11-04 09:33:07,304 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,309 [292] [DEBUG] [auth.credentials] Successfully validated credentials for access token 3 gunicorn-registry stdout | 2025-11-04 09:33:07,309 [292] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [None, None, , None, None, None, None]) gunicorn-registry stdout | 2025-11-04 09:33:07,309 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:07,309 [292] [DEBUG] [auth.permissions] Loading permissions for token: K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X gunicorn-registry stdout | 2025-11-04 09:33:07,310 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,314 [292] [DEBUG] [auth.permissions] Delegate token added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='write') gunicorn-registry stdout | 2025-11-04 09:33:07,314 [292] [DEBUG] [endpoints.v2.v2auth] Request audience: quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com gunicorn-registry stdout | 2025-11-04 09:33:07,314 [292] [DEBUG] [endpoints.v2.v2auth] Scope request: ['repository:build_test/upload_file_firefox:pull'] gunicorn-registry stdout | 2025-11-04 09:33:07,314 [292] [DEBUG] [endpoints.v2.v2auth] Match: ('build_test/upload_file_firefox', 'build_test/upload_file_firefox', 'pull') gunicorn-registry stdout | 2025-11-04 09:33:07,315 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,318 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,322 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,325 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,328 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,373 [292] [DEBUG] [app] Ending request: urn:request:477c49c6-627f-480b-9607-45bb14168352 (/v2/auth) {'endpoint': 'v2.generate_registry_jwt', 'request_id': 'urn:request:477c49c6-627f-480b-9607-45bb14168352', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/auth?account=$token&scope=repository:build_test/upload_file_firefox:pull&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com', 'path': '/v2/auth', 'parameters': {'account': '$token', 'scope': 'repository:build_test/upload_file_firefox:pull', 'service': 'quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com'}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:07,373 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:07,374 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:07,374 [292] [INFO] [gunicorn.access] 10.131.0.6 - $token [04/Nov/2025:09:33:07 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1242 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - $token [04/Nov/2025:09:33:07 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1242 "-" "containers/5.29.5 (github.com/containers/image)" (0.071 487 0.071) gunicorn-registry stdout | 2025-11-04 09:33:07,385 [292] [DEBUG] [app] Starting request: urn:request:fd62de62-4d9e-4ddd-98aa-06f17265b3a7 (/v2/build_test/upload_file_firefox/manifests/latest) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:07,385 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'manifest_ref': 'latest', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:07,385 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg3ODcsImlhdCI6MTc2MjI0ODc4NywiZXhwIjoxNzYyMjUyMzg3LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJ2ZXJzaW9uIjoyLCJlbnRpdHlfa2luZCI6InRva2VuIiwiZW50aXR5X3JlZmVyZW5jZSI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJraW5kIjoidG9rZW4iLCJ0b2tlbiI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCI6IiRkaXNhYmxlZCJ9LCJjb20uYXBvc3RpbGxlLnJvb3QiOiIkZGlzYWJsZWQifX0.QuIQiOD4c5R_l54dkw2CsqXpw_9Y2m6WJ52xxbHNkoup7pbCQhhTe8kH0jAfsR_SvGuKMPrtLU5AoWwaC89ZiEegoNkE-oSn66hpAubQwzDxLkxITTSrMWkK-pyIfLP_oEXw_MipMg8HXzF07UyMWWAkErv0M8fa24nZZ_jgAIBcR0HSvqXuxREJP4mDvSvvohlP0iq80YUv7XrBQFosSh7OIERPdHUjnl8RpzB3GMP4MRTMRnzrbL8j52eyRvokY3KVpa4_UKhydk4qTv4ApYta9sWtusuJEp4j284AUi23fHd3P_P0bstPwXTJY1iAIyXjSsehImH2Z9p_Xn-VHw gunicorn-registry stdout | 2025-11-04 09:33:07,385 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg3ODcsImlhdCI6MTc2MjI0ODc4NywiZXhwIjoxNzYyMjUyMzg3LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJ2ZXJzaW9uIjoyLCJlbnRpdHlfa2luZCI6InRva2VuIiwiZW50aXR5X3JlZmVyZW5jZSI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJraW5kIjoidG9rZW4iLCJ0b2tlbiI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCI6IiRkaXNhYmxlZCJ9LCJjb20uYXBvc3RpbGxlLnJvb3QiOiIkZGlzYWJsZWQifX0.QuIQiOD4c5R_l54dkw2CsqXpw_9Y2m6WJ52xxbHNkoup7pbCQhhTe8kH0jAfsR_SvGuKMPrtLU5AoWwaC89ZiEegoNkE-oSn66hpAubQwzDxLkxITTSrMWkK-pyIfLP_oEXw_MipMg8HXzF07UyMWWAkErv0M8fa24nZZ_jgAIBcR0HSvqXuxREJP4mDvSvvohlP0iq80YUv7XrBQFosSh7OIERPdHUjnl8RpzB3GMP4MRTMRnzrbL8j52eyRvokY3KVpa4_UKhydk4qTv4ApYta9sWtusuJEp4j284AUi23fHd3P_P0bstPwXTJY1iAIyXjSsehImH2Z9p_Xn-VHw gunicorn-registry stdout | 2025-11-04 09:33:07,386 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 33, 7, 385513), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 33, 7, 385538), 'quay', None, datetime.datetime(2025, 11, 3, 9, 33, 7, 385552), datetime.datetime(2025, 10, 28, 9, 33, 7, 385564), None]) gunicorn-registry stdout | 2025-11-04 09:33:07,390 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:07,390 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:07,390 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:07,390 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:07,390 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:07,391 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,396 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,399 [292] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['build_test', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:33:07,399 [292] [DEBUG] [data.cache.cache_key] Loading repository lookup from cache_key: repository_lookup_build_test_upload_file_firefox_latest gunicorn-registry stdout | 2025-11-04 09:33:07,399 [292] [DEBUG] [data.cache.impl] Checking cache for key repository_lookup_build_test_upload_file_firefox_latest gunicorn-registry stdout | 2025-11-04 09:33:07,399 [292] [DEBUG] [data.cache.impl] Found no result in cache for key repository_lookup_build_test_upload_file_firefox_latest; calling loader gunicorn-registry stdout | 2025-11-04 09:33:07,400 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,403 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,406 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,409 [292] [DEBUG] [data.cache.impl] Got loaded result for key repository_lookup_build_test_upload_file_firefox_latest: {'id': 2, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}} gunicorn-registry stdout | 2025-11-04 09:33:07,409 [292] [DEBUG] [data.cache.impl] Caching loaded result for key repository_lookup_build_test_upload_file_firefox_latest with expiration {'id': 2, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:33:07,409 [292] [DEBUG] [data.cache.impl] Cached loaded result for key repository_lookup_build_test_upload_file_firefox_latest with expiration {'id': 2, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:33:07,410 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id", "t2"."id", "t2"."repository_id", "t2"."digest", "t2"."media_type_id", "t2"."manifest_bytes", "t2"."config_media_type", "t2"."layers_compressed_size", "t2"."subject", "t2"."subject_backfilled", "t2"."artifact_type", "t2"."artifact_type_backfilled" FROM "tag" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t1"."manifest_id" = "t2"."id") WHERE (((("t1"."repository_id" = %s) AND ("t1"."name" = %s)) AND (("t1"."lifetime_end_ms" IS %s) OR ("t1"."lifetime_end_ms" > %s))) AND ("t1"."hidden" = %s)) LIMIT %s OFFSET %s', [2, 'latest', None, 1762248787410, False, 1, 0]) quotatotalworker stdout | 2025-11-04 09:33:07,410 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:07,414 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "mediatype" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:33:07,418 [292] [DEBUG] [util.audit] Checking publishing pull_repo to the user events system gunicorn-registry stdout | 2025-11-04 09:33:07,418 [292] [DEBUG] [util.audit] Resolving IP address 10.131.0.6 gunicorn-registry stdout | 2025-11-04 09:33:07,419 [292] [DEBUG] [util.audit] Resolved IP address 10.131.0.6 gunicorn-registry stdout | 2025-11-04 09:33:07,419 [292] [DEBUG] [util.audit] Logging the pull_repo to logs system gunicorn-registry stdout | 2025-11-04 09:33:07,419 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,422 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "logentrykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:33:07,430 [292] [DEBUG] [peewee] ('INSERT INTO "logentry3" ("kind_id", "account_id", "performer_id", "repository_id", "datetime", "ip", "metadata_json") VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING "logentry3"."id"', [41, 2, None, 2, datetime.datetime(2025, 11, 4, 9, 33, 7, 419295), '10.131.0.6', '{"repo": "upload_file_firefox", "namespace": "build_test", "user-agent": "containers/5.29.5 (github.com/containers/image)", "tag": "latest", "token": "Repository Build Token", "resolved_ip": {"provider": "internet", "service": null, "sync_token": "1645662201", "country_iso_code": null, "aws_region": null, "continent": null}}']) gunicorn-registry stdout | 2025-11-04 09:33:07,434 [292] [DEBUG] [util.audit] Track and log of pull_repo complete gunicorn-registry stdout | 2025-11-04 09:33:07,435 [292] [DEBUG] [app] Ending request: urn:request:fd62de62-4d9e-4ddd-98aa-06f17265b3a7 (/v2/build_test/upload_file_firefox/manifests/latest) {'endpoint': 'v2.fetch_manifest_by_tagname', 'request_id': 'urn:request:fd62de62-4d9e-4ddd-98aa-06f17265b3a7', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/manifests/latest', 'path': '/v2/build_test/upload_file_firefox/manifests/latest', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:07,435 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:07,435 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:07,435 [292] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:07 +0000] "GET /v2/build_test/upload_file_firefox/manifests/latest HTTP/1.1" 200 777 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:07 +0000] "GET /v2/build_test/upload_file_firefox/manifests/latest HTTP/1.1" 200 777 "-" "containers/5.29.5 (github.com/containers/image)" (0.051 1902 0.052) gunicorn-registry stdout | 2025-11-04 09:33:07,438 [292] [DEBUG] [app] Starting request: urn:request:55073b58-c698-4435-878f-01158c54a72c (/v2/build_test/upload_file_firefox/blobs/sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:07,439 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'digest': 'sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:07,439 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg3ODcsImlhdCI6MTc2MjI0ODc4NywiZXhwIjoxNzYyMjUyMzg3LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJ2ZXJzaW9uIjoyLCJlbnRpdHlfa2luZCI6InRva2VuIiwiZW50aXR5X3JlZmVyZW5jZSI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJraW5kIjoidG9rZW4iLCJ0b2tlbiI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCI6IiRkaXNhYmxlZCJ9LCJjb20uYXBvc3RpbGxlLnJvb3QiOiIkZGlzYWJsZWQifX0.QuIQiOD4c5R_l54dkw2CsqXpw_9Y2m6WJ52xxbHNkoup7pbCQhhTe8kH0jAfsR_SvGuKMPrtLU5AoWwaC89ZiEegoNkE-oSn66hpAubQwzDxLkxITTSrMWkK-pyIfLP_oEXw_MipMg8HXzF07UyMWWAkErv0M8fa24nZZ_jgAIBcR0HSvqXuxREJP4mDvSvvohlP0iq80YUv7XrBQFosSh7OIERPdHUjnl8RpzB3GMP4MRTMRnzrbL8j52eyRvokY3KVpa4_UKhydk4qTv4ApYta9sWtusuJEp4j284AUi23fHd3P_P0bstPwXTJY1iAIyXjSsehImH2Z9p_Xn-VHw gunicorn-registry stdout | 2025-11-04 09:33:07,439 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg3ODcsImlhdCI6MTc2MjI0ODc4NywiZXhwIjoxNzYyMjUyMzg3LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJ2ZXJzaW9uIjoyLCJlbnRpdHlfa2luZCI6InRva2VuIiwiZW50aXR5X3JlZmVyZW5jZSI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJraW5kIjoidG9rZW4iLCJ0b2tlbiI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCI6IiRkaXNhYmxlZCJ9LCJjb20uYXBvc3RpbGxlLnJvb3QiOiIkZGlzYWJsZWQifX0.QuIQiOD4c5R_l54dkw2CsqXpw_9Y2m6WJ52xxbHNkoup7pbCQhhTe8kH0jAfsR_SvGuKMPrtLU5AoWwaC89ZiEegoNkE-oSn66hpAubQwzDxLkxITTSrMWkK-pyIfLP_oEXw_MipMg8HXzF07UyMWWAkErv0M8fa24nZZ_jgAIBcR0HSvqXuxREJP4mDvSvvohlP0iq80YUv7XrBQFosSh7OIERPdHUjnl8RpzB3GMP4MRTMRnzrbL8j52eyRvokY3KVpa4_UKhydk4qTv4ApYta9sWtusuJEp4j284AUi23fHd3P_P0bstPwXTJY1iAIyXjSsehImH2Z9p_Xn-VHw gunicorn-registry stdout | 2025-11-04 09:33:07,439 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:07,439 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:07,439 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:07,440 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:07,440 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:07,440 [292] [DEBUG] [data.cache.impl] Checking cache for key geo_restrictions__build_test gunicorn-registry stdout | 2025-11-04 09:33:07,440 [292] [DEBUG] [data.cache.impl] Found no result in cache for key geo_restrictions__build_test; calling loader gunicorn-registry stdout | 2025-11-04 09:33:07,440 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."added", "t1"."description", "t1"."unstructured_json", "t1"."restricted_region_iso_code" FROM "namespacegeorestriction" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:07,445 [292] [DEBUG] [data.cache.impl] Got loaded result for key geo_restrictions__build_test: [] gunicorn-registry stdout | 2025-11-04 09:33:07,445 [292] [DEBUG] [data.cache.impl] Caching loaded result for key geo_restrictions__build_test with expiration []: 240s gunicorn-registry stdout | 2025-11-04 09:33:07,445 [292] [DEBUG] [data.cache.impl] Cached loaded result for key geo_restrictions__build_test with expiration []: 240s gunicorn-registry stdout | 2025-11-04 09:33:07,446 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,450 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,453 [292] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['build_test', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:33:07,453 [292] [DEBUG] [data.cache.impl] Checking cache for key repo_blob__build_test_upload_file_firefox_sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834_2 gunicorn-registry stdout | 2025-11-04 09:33:07,453 [292] [DEBUG] [data.cache.impl] Found no result in cache for key repo_blob__build_test_upload_file_firefox_sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834_2; calling loader gunicorn-registry stdout | 2025-11-04 09:33:07,454 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,457 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,460 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,464 [292] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,467 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['f1c69190-31f9-4502-a187-b21c65393738']) gunicorn-registry stdout | 2025-11-04 09:33:07,470 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['f1c69190-31f9-4502-a187-b21c65393738']) gunicorn-registry stdout | 2025-11-04 09:33:07,473 [292] [DEBUG] [data.cache.impl] Got loaded result for key repo_blob__build_test_upload_file_firefox_sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834_2: {'uuid': 'f1c69190-31f9-4502-a187-b21c65393738', 'digest': 'sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834', 'compressed_size': 7830, 'uncompressed_size': None, 'uploading': True, 'db_id': 3, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/ea/ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834'}} gunicorn-registry stdout | 2025-11-04 09:33:07,473 [292] [DEBUG] [data.cache.impl] Caching loaded result for key repo_blob__build_test_upload_file_firefox_sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834_2 with expiration {'uuid': 'f1c69190-31f9-4502-a187-b21c65393738', 'digest': 'sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834', 'compressed_size': 7830, 'uncompressed_size': None, 'uploading': True, 'db_id': 3, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/ea/ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834'}}: 60s gunicorn-registry stdout | 2025-11-04 09:33:07,473 [292] [DEBUG] [data.cache.impl] Cached loaded result for key repo_blob__build_test_upload_file_firefox_sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834_2 with expiration {'uuid': 'f1c69190-31f9-4502-a187-b21c65393738', 'digest': 'sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834', 'compressed_size': 7830, 'uncompressed_size': None, 'uploading': True, 'db_id': 3, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/ea/ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834'}}: 60s gunicorn-registry stdout | 2025-11-04 09:33:07,473 [292] [DEBUG] [endpoints.v2.blob] Looking up the direct download URL for path: sha256/ea/ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834 gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [endpoints.v2.blob] Checking for namespace build_test gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:07,474 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:07,475 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,475 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,475 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:07,475 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | GET gunicorn-registry stdout | /whuaws/quaydata/sha256/ea/ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834 gunicorn-registry stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093307Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | host gunicorn-registry stdout | UNSIGNED-PAYLOAD gunicorn-registry stdout | 2025-11-04 09:33:07,475 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093307Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 4ec51324a847999a5704d815ce07946c3bc171f2d6d4b83e903cce9c1c594a85 gunicorn-registry stdout | 2025-11-04 09:33:07,475 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 90eb692f6aaae32754fdd76ad910cb44f22e17643184666f5cf842787850bd66 gunicorn-registry stdout | 2025-11-04 09:33:07,475 [292] [DEBUG] [endpoints.v2.blob] Returning direct download URL gunicorn-registry stdout | 2025-11-04 09:33:07,476 [292] [DEBUG] [app] Ending request: urn:request:55073b58-c698-4435-878f-01158c54a72c (/v2/build_test/upload_file_firefox/blobs/sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834) {'endpoint': 'v2.download_blob', 'request_id': 'urn:request:55073b58-c698-4435-878f-01158c54a72c', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834', 'path': '/v2/build_test/upload_file_firefox/blobs/sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:07,476 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:07,476 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:07,476 [292] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:07 +0000] "GET /v2/build_test/upload_file_firefox/blobs/sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834 HTTP/1.1" 302 1009 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:07 +0000] "GET /v2/build_test/upload_file_firefox/blobs/sha256:ea8bb5ed4de2c6e8c1d56e36eeb233d17f693203c966315fa471a0fe48b48834 HTTP/1.1" 302 1009 "-" "containers/5.29.5 (github.com/containers/image)" (0.038 1604 0.038) builder stdout | 2025-11-04 09:33:07,666 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:33:07,679 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:33:07,679 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:33:07,689 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:07,709 [292] [DEBUG] [app] Starting request: urn:request:0a9e8818-ac87-4bf5-ae01-d2eb98d0a6f1 (/v2/build_test/upload_file_firefox/blobs/sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:07,709 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'digest': 'sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:07,709 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg3ODcsImlhdCI6MTc2MjI0ODc4NywiZXhwIjoxNzYyMjUyMzg3LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJ2ZXJzaW9uIjoyLCJlbnRpdHlfa2luZCI6InRva2VuIiwiZW50aXR5X3JlZmVyZW5jZSI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJraW5kIjoidG9rZW4iLCJ0b2tlbiI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCI6IiRkaXNhYmxlZCJ9LCJjb20uYXBvc3RpbGxlLnJvb3QiOiIkZGlzYWJsZWQifX0.QuIQiOD4c5R_l54dkw2CsqXpw_9Y2m6WJ52xxbHNkoup7pbCQhhTe8kH0jAfsR_SvGuKMPrtLU5AoWwaC89ZiEegoNkE-oSn66hpAubQwzDxLkxITTSrMWkK-pyIfLP_oEXw_MipMg8HXzF07UyMWWAkErv0M8fa24nZZ_jgAIBcR0HSvqXuxREJP4mDvSvvohlP0iq80YUv7XrBQFosSh7OIERPdHUjnl8RpzB3GMP4MRTMRnzrbL8j52eyRvokY3KVpa4_UKhydk4qTv4ApYta9sWtusuJEp4j284AUi23fHd3P_P0bstPwXTJY1iAIyXjSsehImH2Z9p_Xn-VHw gunicorn-registry stdout | 2025-11-04 09:33:07,709 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg3ODcsImlhdCI6MTc2MjI0ODc4NywiZXhwIjoxNzYyMjUyMzg3LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJ2ZXJzaW9uIjoyLCJlbnRpdHlfa2luZCI6InRva2VuIiwiZW50aXR5X3JlZmVyZW5jZSI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJraW5kIjoidG9rZW4iLCJ0b2tlbiI6IkswVlFZWUJZSE5aT1RESUc3OEEzQUo5UEpXNUwzSDBVMjBTU1cwMlhESVRMMFBWSDUzM01OSUZTQk01R0RMOVgiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCI6IiRkaXNhYmxlZCJ9LCJjb20uYXBvc3RpbGxlLnJvb3QiOiIkZGlzYWJsZWQifX0.QuIQiOD4c5R_l54dkw2CsqXpw_9Y2m6WJ52xxbHNkoup7pbCQhhTe8kH0jAfsR_SvGuKMPrtLU5AoWwaC89ZiEegoNkE-oSn66hpAubQwzDxLkxITTSrMWkK-pyIfLP_oEXw_MipMg8HXzF07UyMWWAkErv0M8fa24nZZ_jgAIBcR0HSvqXuxREJP4mDvSvvohlP0iq80YUv7XrBQFosSh7OIERPdHUjnl8RpzB3GMP4MRTMRnzrbL8j52eyRvokY3KVpa4_UKhydk4qTv4ApYta9sWtusuJEp4j284AUi23fHd3P_P0bstPwXTJY1iAIyXjSsehImH2Z9p_Xn-VHw gunicorn-registry stdout | 2025-11-04 09:33:07,710 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:07,710 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:07,710 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:07,710 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:07,710 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:07,710 [292] [DEBUG] [data.cache.impl] Checking cache for key geo_restrictions__build_test gunicorn-registry stdout | 2025-11-04 09:33:07,711 [292] [DEBUG] [data.cache.impl] Found result in cache for key geo_restrictions__build_test: [] gunicorn-registry stdout | 2025-11-04 09:33:07,712 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,716 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,719 [292] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['build_test', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:33:07,719 [292] [DEBUG] [data.cache.impl] Checking cache for key repo_blob__build_test_upload_file_firefox_sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f_2 gunicorn-registry stdout | 2025-11-04 09:33:07,719 [292] [DEBUG] [data.cache.impl] Found no result in cache for key repo_blob__build_test_upload_file_firefox_sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f_2; calling loader gunicorn-registry stdout | 2025-11-04 09:33:07,720 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,723 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,726 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,730 [292] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:07,733 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['7ec99d5f-7796-490d-96b0-ab7ce5fbd85f']) gunicorn-registry stdout | 2025-11-04 09:33:07,736 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['7ec99d5f-7796-490d-96b0-ab7ce5fbd85f']) gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [data.cache.impl] Got loaded result for key repo_blob__build_test_upload_file_firefox_sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f_2: {'uuid': '7ec99d5f-7796-490d-96b0-ab7ce5fbd85f', 'digest': 'sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f', 'compressed_size': 20144818, 'uncompressed_size': 32308224, 'uploading': True, 'db_id': 1, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/a0/a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f'}} gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [data.cache.impl] Caching loaded result for key repo_blob__build_test_upload_file_firefox_sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f_2 with expiration {'uuid': '7ec99d5f-7796-490d-96b0-ab7ce5fbd85f', 'digest': 'sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f', 'compressed_size': 20144818, 'uncompressed_size': 32308224, 'uploading': True, 'db_id': 1, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/a0/a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f'}}: 60s gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [data.cache.impl] Cached loaded result for key repo_blob__build_test_upload_file_firefox_sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f_2 with expiration {'uuid': '7ec99d5f-7796-490d-96b0-ab7ce5fbd85f', 'digest': 'sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f', 'compressed_size': 20144818, 'uncompressed_size': 32308224, 'uploading': True, 'db_id': 1, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/a0/a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f'}}: 60s gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [endpoints.v2.blob] Looking up the direct download URL for path: sha256/a0/a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [endpoints.v2.blob] Checking for namespace build_test gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,739 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | GET gunicorn-registry stdout | /whuaws/quaydata/sha256/a0/a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f gunicorn-registry stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093307Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | host gunicorn-registry stdout | UNSIGNED-PAYLOAD gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093307Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | afbabe2f673212e68737d8cd3a82b6fc0494d62838a754e69dbf5bf17bf7d2cb gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 059ec29d240992003f3e6de6a676d7c0cb43bba95dd6ff62ce8bc073cf8ee9ed gunicorn-registry stdout | 2025-11-04 09:33:07,740 [292] [DEBUG] [endpoints.v2.blob] Returning direct download URL gunicorn-registry stdout | 2025-11-04 09:33:07,741 [292] [DEBUG] [app] Ending request: urn:request:0a9e8818-ac87-4bf5-ae01-d2eb98d0a6f1 (/v2/build_test/upload_file_firefox/blobs/sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f) {'endpoint': 'v2.download_blob', 'request_id': 'urn:request:0a9e8818-ac87-4bf5-ae01-d2eb98d0a6f1', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f', 'path': '/v2/build_test/upload_file_firefox/blobs/sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:07,741 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:07,741 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:07,741 [292] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:07 +0000] "GET /v2/build_test/upload_file_firefox/blobs/sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f HTTP/1.1" 302 1009 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:07 +0000] "GET /v2/build_test/upload_file_firefox/blobs/sha256:a0ef554eda891755dbfcf723f1a83802e418e13a01455ef0985f52919c59bd8f HTTP/1.1" 302 1009 "-" "containers/5.29.5 (github.com/containers/image)" (0.033 1604 0.034) builder stdout | 2025-11-04 09:33:07,814 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:07.809082\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-registry stdout | 2025-11-04 09:33:08,426 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:08,431 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:33:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:33:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:21.434460+00:00 (in 12.998628 seconds) namespacegcworker stdout | 2025-11-04 09:33:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:38:08 GMT)" (scheduled at 2025-11-04 09:33:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:33:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 33, 8, 436084), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:33:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:33:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:35.435809+00:00 (in 26.998359 seconds) gcworker stdout | 2025-11-04 09:33:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:38 GMT)" (scheduled at 2025-11-04 09:33:08.437104+00:00) gcworker stdout | 2025-11-04 09:33:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:08,442 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:33:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 33, 8, 436084), True, datetime.datetime(2025, 11, 4, 9, 33, 8, 436084), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:33:08,448 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248728448, None, 1, 0]) namespacegcworker stdout | 2025-11-04 09:33:08,451 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 33, 8, 436084), True, datetime.datetime(2025, 11, 4, 9, 33, 8, 436084), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 33, 8, 436084), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:33:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:33:08,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:33:08,455 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:33:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:38:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:33:08,455 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:08,480 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:08,494 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:08,525 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:08,679 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:08,681 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:33:08,833 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:33:08,834 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:33:08,836 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:33:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:33:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:57.131795+00:00 (in 47.997730 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:09 GMT)" (scheduled at 2025-11-04 09:33:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:33:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:33:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:33:09,537 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:09,827 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:09.819745\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} manifestbackfillworker stdout | 2025-11-04 09:33:11,305 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:33:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:33:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:15.535095+00:00 (in 4.004138 seconds) notificationworker stdout | 2025-11-04 09:33:11,531 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:21 GMT)" (scheduled at 2025-11-04 09:33:11.530064+00:00) notificationworker stdout | 2025-11-04 09:33:11,531 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:33:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 11, 531260), True, datetime.datetime(2025, 11, 4, 9, 33, 11, 531260), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:33:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:33:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:33:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:21 GMT)" executed successfully builder stdout | 2025-11-04 09:33:11,766 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:11,782 [66] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:11 +0000] "POST /buildman_pb.BuildManager/SetPhase HTTP/2.0" 200 9 "-" "grpc-go/1.71.0" (0.034 998 0.034) builder stdout | 2025-11-04 09:33:11,798 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:11.792840\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:11,837 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:11.831382\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} quotaregistrysizeworker stdout | 2025-11-04 09:33:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:33:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:12.222828+00:00 (in 59.999579 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:33:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:12 GMT)" (scheduled at 2025-11-04 09:33:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:33:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:33:12,235 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:33:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:33:12,756 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:13,515 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.509366\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,525 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.518305\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,533 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.528158\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,784 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.778728\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,792 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.787277\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,800 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.795161\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,809 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.802867\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,817 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.811690\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,825 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.820655\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,833 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.828302\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,841 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.836017\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,846 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.840996\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,850 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.845221\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,857 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.852807\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,865 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.860294\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,873 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.868297\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,881 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.876221\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,889 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.884276\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,896 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.891827\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,903 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.899102\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,911 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.906411\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,919 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.914028\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,926 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.921765\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,934 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.929275\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,941 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.936818\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,948 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.944130\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,956 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.951526\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,963 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.958950\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:13,971 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:13.966276\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:14,329 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:14.323951\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} quotatotalworker stdout | 2025-11-04 09:33:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:33:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:29.351216+00:00 (in 14.999588 seconds) quotatotalworker stdout | 2025-11-04 09:33:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:29 GMT)" (scheduled at 2025-11-04 09:33:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:33:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:33:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:33:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:33:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:33:15,262 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:33:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:33:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:17.479080+00:00 (in 1.998871 seconds) securityworker stdout | 2025-11-04 09:33:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:45 GMT)" (scheduled at 2025-11-04 09:33:15.479769+00:00) securityworker stdout | 2025-11-04 09:33:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:33:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:33:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:33:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:33:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:33:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:33:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 2]) securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:33:15,500 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 28, 15, 484088), 1, 2]) securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:33:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:33:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:33:15,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:15,505 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 28, 15, 484088), 1, 2]) securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-2 by worker securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 2 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] Total range: 2-1 securityworker stderr | 2025-11-04 09:33:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:33:15,508 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:33:15,508 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:33:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:33:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:21.530064+00:00 (in 5.994535 seconds) notificationworker stdout | 2025-11-04 09:33:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:15 GMT)" (scheduled at 2025-11-04 09:33:15.535095+00:00) notificationworker stdout | 2025-11-04 09:33:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:33:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:33:15,697 [288] [DEBUG] [app] Starting request: urn:request:22385621-cb6b-4dc4-b89f-03c093061619 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:15,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:15,701 [292] [DEBUG] [app] Starting request: urn:request:153e63f1-c41e-497d-9fc9-d37f65d59726 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:15,702 [292] [DEBUG] [app] Ending request: urn:request:153e63f1-c41e-497d-9fc9-d37f65d59726 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:153e63f1-c41e-497d-9fc9-d37f65d59726', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:33:15,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:33:15,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:15,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:15,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:15,706 [285] [DEBUG] [app] Starting request: urn:request:15496663-66f4-4565-b995-b4d2e3a41def (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:15,706 [285] [DEBUG] [app] Ending request: urn:request:15496663-66f4-4565-b995-b4d2e3a41def (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:15496663-66f4-4565-b995-b4d2e3a41def', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:33:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:33:15,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:15,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:15,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:15,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:15,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:15,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:15,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:15,731 [288] [DEBUG] [app] Ending request: urn:request:22385621-cb6b-4dc4-b89f-03c093061619 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:22385621-cb6b-4dc4-b89f-03c093061619', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:15,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:15,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:33:15,783 [288] [DEBUG] [app] Starting request: urn:request:015cf420-af3a-4cf1-8554-d4d294d1044a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:15,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:15,785 [292] [DEBUG] [app] Starting request: urn:request:806f3c23-914e-42de-9379-d7ddfcf52514 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:15,785 [292] [DEBUG] [app] Ending request: urn:request:806f3c23-914e-42de-9379-d7ddfcf52514 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:806f3c23-914e-42de-9379-d7ddfcf52514', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:33:15,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:33:15,786 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:15,787 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:15,789 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:15,790 [285] [DEBUG] [app] Starting request: urn:request:0c1a788b-3400-4a8f-b0d8-bcfa294a5f3b (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:15,790 [285] [DEBUG] [app] Ending request: urn:request:0c1a788b-3400-4a8f-b0d8-bcfa294a5f3b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0c1a788b-3400-4a8f-b0d8-bcfa294a5f3b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:33:15,790 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:33:15,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:15,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:15,791 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:15,791 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:15,798 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:15,798 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:15,807 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:15,811 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:15,814 [288] [DEBUG] [app] Ending request: urn:request:015cf420-af3a-4cf1-8554-d4d294d1044a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:015cf420-af3a-4cf1-8554-d4d294d1044a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:15,814 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:15,814 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) builder stdout | 2025-11-04 09:33:15,858 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:15.850350\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,256 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.250590\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,264 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.259415\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,273 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.267215\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,281 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.276520\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,291 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.287142\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,310 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.305338\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,318 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.313177\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,328 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.320720\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,337 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.330749\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,344 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.339757\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,516 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.510095\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,524 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.518947\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,531 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.526791\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,540 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.534352\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,549 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.543530\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,556 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.551570\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,563 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.559003\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,571 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.566310\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,579 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.573961\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,586 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.581589\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,594 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.589147\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,602 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.597165\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,610 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.604954\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,649 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.644034\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,657 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.652315\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,681 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.676766\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,689 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.684583\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,697 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.692182\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,704 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.699830\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,712 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.707396\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,719 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.715133\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,727 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.722746\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,735 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.730249\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,743 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.738054\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,750 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.745711\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,758 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.753285\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,765 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.761199\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,773 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.768543\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,781 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.776120\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,788 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.784005\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,796 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.791737\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,803 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.799173\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,811 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.806481\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,819 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.814273\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,829 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.822572\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,838 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.832574\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:16,849 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:16.841523\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,041 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.036088\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,050 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.045060\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,058 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.053176\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,066 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.060915\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,074 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.069000\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,082 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.077140\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,089 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.084998\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,097 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.092622\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,105 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.100178\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,113 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.107915\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,120 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.115641\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} proxycacheblobworker stdout | 2025-11-04 09:33:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:33:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:27.122849+00:00 (in 9.999640 seconds) proxycacheblobworker stdout | 2025-11-04 09:33:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:27 GMT)" (scheduled at 2025-11-04 09:33:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:33:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:33:17,123 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 17, 123434), True, datetime.datetime(2025, 11, 4, 9, 33, 17, 123434), 0, 'proxycacheblob/%', 50, 1, 0]) builder stdout | 2025-11-04 09:33:17,128 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.123390\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} proxycacheblobworker stdout | 2025-11-04 09:33:17,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:33:17,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:33:17,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:27 GMT)" executed successfully builder stdout | 2025-11-04 09:33:17,136 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.130927\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,143 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.138758\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,151 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.146343\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,159 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.153829\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,167 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.161982\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,175 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.170010\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,186 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.177554\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,193 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.188857\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,203 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.196308\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,211 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.205825\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,221 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.213804\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,229 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.223641\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,236 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.231693\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,244 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.239318\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,253 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.247295\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:17,354 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.348305\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} securityworker stdout | 2025-11-04 09:33:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:33:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:45.479769+00:00 (in 28.000234 seconds) securityworker stdout | 2025-11-04 09:33:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:47 GMT)" (scheduled at 2025-11-04 09:33:17.479080+00:00) securityworker stdout | 2025-11-04 09:33:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:33:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:33:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:33:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:33:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:33:17,495 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:47 GMT)" executed successfully builder stdout | 2025-11-04 09:33:17,689 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:33:17,702 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:33:17,702 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:33:17,712 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:17,868 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:17.861913\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} securityscanningnotificationworker stdout | 2025-11-04 09:33:18,198 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:33:18,871 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:19,280 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.273701\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,292 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.283300\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,300 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.295420\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,308 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.303427\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,316 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.311174\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,323 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.318924\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,331 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.326538\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,339 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.334263\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,346 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.341943\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,354 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.349227\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,361 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.356527\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,368 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.363928\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,376 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.371184\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,383 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.378626\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,391 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.386361\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,398 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.393764\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,410 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.405894\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,418 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.413329\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,425 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.421124\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,433 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.428388\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,742 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.736746\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,749 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.744851\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,757 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.752572\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,765 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.760295\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,773 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.768441\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,781 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.776254\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,800 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.794583\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,859 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.854345\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,874 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.869889\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:19,877 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:19.871027\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} repositorygcworker stdout | 2025-11-04 09:33:20,180 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:20,608 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.602143\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,621 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.616141\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,814 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.808580\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,878 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.872718\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,885 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.880942\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,893 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.888594\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,901 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.896409\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,911 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.906000\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,919 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.913877\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,927 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.922590\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,935 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.930100\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,964 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.958852\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,972 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.967025\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,979 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.974690\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,987 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.982756\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:20,995 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.990581\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,003 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:20.998215\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,011 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.005767\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,019 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.013745\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,027 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.022338\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,034 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.029978\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,043 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.037479\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,053 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.046030\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,063 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.055945\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,071 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.066070\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,079 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.074354\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,087 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.082189\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,095 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.089906\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,102 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.097596\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,110 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.105651\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,118 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.113635\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,127 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.121969\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,135 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.129940\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} autopruneworker stdout | 2025-11-04 09:33:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:33:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:51.226294+00:00 (in 29.999604 seconds) autopruneworker stdout | 2025-11-04 09:33:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:51 GMT)" (scheduled at 2025-11-04 09:33:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:33:21,234 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248741234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:33:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:33:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:33:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:51 GMT)" executed successfully builder stdout | 2025-11-04 09:33:21,317 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.311560\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,327 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.320027\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,335 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.329928\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,344 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.337997\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,352 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.347046\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,360 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.355002\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,367 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.362721\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,375 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.370241\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,384 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.378118\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,392 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.386787\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,403 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.394831\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,414 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.406605\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,424 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.416853\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,432 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.427443\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} namespacegcworker stdout | 2025-11-04 09:33:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:33:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:32.435813+00:00 (in 11.001001 seconds) namespacegcworker stdout | 2025-11-04 09:33:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:21 GMT)" (scheduled at 2025-11-04 09:33:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:33:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:33:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 21, 435103), True, datetime.datetime(2025, 11, 4, 9, 33, 21, 435103), 0, 'namespacegc/%', 50, 1, 0]) builder stdout | 2025-11-04 09:33:21,440 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.435473\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} namespacegcworker stdout | 2025-11-04 09:33:21,446 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:33:21,446 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:33:21,446 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:21 GMT)" executed successfully builder stdout | 2025-11-04 09:33:21,448 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.443372\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,456 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.451232\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,464 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.458896\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,472 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.467207\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,479 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.474877\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,511 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.506144\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,518 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.514028\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,526 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.521693\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} notificationworker stdout | 2025-11-04 09:33:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:33:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:31.530064+00:00 (in 9.999616 seconds) notificationworker stdout | 2025-11-04 09:33:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:31 GMT)" (scheduled at 2025-11-04 09:33:21.530064+00:00) notificationworker stdout | 2025-11-04 09:33:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:33:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 21, 530709), True, datetime.datetime(2025, 11, 4, 9, 33, 21, 530709), 0, 'notification/%', 50, 1, 0]) builder stdout | 2025-11-04 09:33:21,534 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.529359\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,541 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.536952\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} notificationworker stdout | 2025-11-04 09:33:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:33:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:33:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:31 GMT)" executed successfully builder stdout | 2025-11-04 09:33:21,549 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.544480\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,557 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.552291\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,564 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.559843\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,572 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.567766\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,581 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.575523\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,589 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.584672\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,597 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.592271\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,605 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.600179\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,613 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.607839\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,620 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.615835\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,628 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.623554\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,637 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.631506\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,645 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.640859\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,653 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.648725\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,661 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.656522\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,669 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.664560\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,677 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.672298\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,684 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.679870\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,692 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.687605\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,700 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.695538\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,708 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.703290\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,716 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.711230\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,724 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.719122\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,731 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.726969\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,739 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.734553\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,746 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.742082\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,754 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.749662\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,762 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.757409\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,839 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.833169\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,847 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.842159\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,855 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.850359\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,863 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.858282\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,871 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.866389\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,879 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.874257\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,885 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.880422\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,887 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.882510\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,895 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.890668\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,903 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.898421\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,912 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.906085\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,921 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.915637\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,930 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.924406\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,939 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.933704\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,947 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.942183\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,955 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.950022\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,963 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.958135\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,978 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.966318\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,986 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.981341\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:21,994 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.989282\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,002 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:21.997097\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,009 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.004977\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,017 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.012797\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,025 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.020798\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,034 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.028740\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,041 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.036670\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,050 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.044526\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,057 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.052695\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,065 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.060590\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,073 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.068645\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,081 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.076402\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,089 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.084005\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,111 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.105762\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,120 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.113793\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,127 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.122964\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,135 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.130516\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,142 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.137992\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,150 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.145545\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,158 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.153473\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,165 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.160923\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,173 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.168706\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,180 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.176175\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,188 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.183475\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,196 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.191391\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,203 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.198704\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,211 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.206556\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,218 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.214003\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,366 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.358741\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,374 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.369173\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,382 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.376997\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,389 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.384652\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,397 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.392202\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,405 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.399856\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,412 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.407841\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,420 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.415152\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,427 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.422899\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,435 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.430171\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,442 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.437992\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,450 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.445592\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,458 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.453074\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} autopruneworker stdout | 2025-11-04 09:33:22,448 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:22,466 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.461212\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,474 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.469042\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,485 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.477116\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,496 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.487877\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,503 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.498909\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,511 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.506571\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,525 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.514402\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,533 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.528720\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,548 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.536553\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,556 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.551620\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,564 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.559518\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,572 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.567373\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,580 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.575390\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,588 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.583392\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,595 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.590984\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,610 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.605859\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,618 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.613692\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,626 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.621446\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,633 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.629250\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,641 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.636753\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,649 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.644553\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,657 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.652313\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,664 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.659887\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,672 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.667719\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,680 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.675671\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,687 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.683108\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,695 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.690437\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,702 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.697757\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,710 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.705396\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-web stdout | 2025-11-04 09:33:22,716 [285] [DEBUG] [app] Starting request: urn:request:53565d85-4ab0-4006-93fc-e73351d2afc2 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'X-Forwarded-For': '10.131.0.6'} gunicorn-web stdout | 2025-11-04 09:33:22,716 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,716 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,717 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) builder stdout | 2025-11-04 09:33:22,718 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.713082\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,725 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.720680\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-web stdout | 2025-11-04 09:33:22,728 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:33:22,728 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:33:22,728 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,729 [285] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:33:22,729 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,729 [285] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,729 [285] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,729 [285] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,730 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) builder stdout | 2025-11-04 09:33:22,733 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.728279\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-web stdout | 2025-11-04 09:33:22,734 [285] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,735 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:22,740 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) builder stdout | 2025-11-04 09:33:22,741 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.736380\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-web stdout | 2025-11-04 09:33:22,744 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,745 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) builder stdout | 2025-11-04 09:33:22,748 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.743850\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-web stdout | 2025-11-04 09:33:22,754 [285] [DEBUG] [app] Ending request: urn:request:53565d85-4ab0-4006-93fc-e73351d2afc2 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'endpoint': 'api.repositorybuildlogs', 'request_id': 'urn:request:53565d85-4ab0-4006-93fc-e73351d2afc2', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=3', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs', 'parameters': {'start': '3'}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:33:22,755 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:22,755 [285] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:22 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=3 HTTP/1.0" 200 84395 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:22 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=3 HTTP/2.0" 200 84395 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.040 1035 0.040) builder stdout | 2025-11-04 09:33:22,758 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.751424\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,765 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.761020\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,773 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.768460\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,782 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.775933\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,789 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.784933\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,797 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.792439\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,805 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.800239\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,813 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.808053\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,820 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.815591\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,828 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.823213\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:22,835 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:22.830848\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-web stdout | 2025-11-04 09:33:22,973 [287] [DEBUG] [app] Starting request: urn:request:3c31026f-2339-4082-b7b7-bfae0c70edae (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'X-Forwarded-For': '10.129.2.10'} gunicorn-web stdout | 2025-11-04 09:33:22,973 [287] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,973 [287] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,974 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,986 [287] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,987 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:22,992 [287] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:33:22,992 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:22,997 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:23,002 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:33:23,008 [287] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:33:23,009 [287] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:23,009 [287] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:23,009 [287] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:33:23,009 [287] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093323Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:33:23,009 [287] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093323Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | 094049304657791c077ba4db8781f82f2b76f2f25ea6a7ba315b2db96b155090 gunicorn-web stdout | 2025-11-04 09:33:23,009 [287] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | 06f5982cccf5a690314994653664985bf8072208360ae00a4c1045e135d7f3c0 gunicorn-web stdout | 2025-11-04 09:33:23,009 [287] [DEBUG] [app] Ending request: urn:request:3c31026f-2339-4082-b7b7-bfae0c70edae (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'endpoint': 'api.repositorybuildresource', 'request_id': 'urn:request:3c31026f-2339-4082-b7b7-bfae0c70edae', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:33:23,010 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:23 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/2.0" 200 889 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.038 1026 0.038) gunicorn-web stdout | 2025-11-04 09:33:23,010 [287] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:23 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/1.0" 200 889 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" builder stdout | 2025-11-04 09:33:23,635 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:23.629070\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:23,643 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:23.638387\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:23,651 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:23.646604\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:23,895 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:23.889800\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} notificationworker stdout | 2025-11-04 09:33:25,307 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:25,750 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:25.744991\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:25,758 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:25.753383\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:25,766 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:25.761257\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:25,774 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:25.768971\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:25,905 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:25.899385\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:26,213 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:26.208066\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:26,234 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:26,250 [66] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:26 +0000] "POST /buildman_pb.BuildManager/SetPhase HTTP/2.0" 200 9 "-" "grpc-go/1.71.0" (0.034 998 0.035) gunicorn-registry stdout | 2025-11-04 09:33:26,295 [292] [DEBUG] [app] Starting request: urn:request:39094fbc-fb0a-4dd7-b4b7-a673f7bca183 (/v2/) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:26,295 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {} gunicorn-registry stdout | 2025-11-04 09:33:26,295 [292] [DEBUG] [auth.registry_jwt_auth] No auth header. gunicorn-registry stdout | 2025-11-04 09:33:26,296 [292] [DEBUG] [app] Ending request: urn:request:39094fbc-fb0a-4dd7-b4b7-a673f7bca183 (/v2/) {'endpoint': 'v2.v2_support_enabled', 'request_id': 'urn:request:39094fbc-fb0a-4dd7-b4b7-a673f7bca183', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/', 'path': '/v2/', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,296 [292] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:26 +0000] "GET /v2/ HTTP/1.1" 401 4 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:26 +0000] "GET /v2/ HTTP/1.1" 401 4 "-" "containers/5.29.5 (github.com/containers/image)" (0.002 243 0.001) gunicorn-registry stdout | 2025-11-04 09:33:26,306 [292] [DEBUG] [app] Starting request: urn:request:0adcf829-f873-44c0-a3d3-55f04dffa39e (/v2/auth) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:26,306 [292] [DEBUG] [auth.basic] Attempt to process basic auth header gunicorn-registry stdout | 2025-11-04 09:33:26,306 [292] [DEBUG] [auth.credentials] Found credentials for access token gunicorn-registry stdout | 2025-11-04 09:33:26,307 [299] [DEBUG] [app] Starting request: urn:request:9de5939c-33fc-45cf-9804-7028339e5828 (/v2/auth) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:26,307 [299] [DEBUG] [auth.basic] Attempt to process basic auth header gunicorn-registry stdout | 2025-11-04 09:33:26,307 [299] [DEBUG] [auth.credentials] Found credentials for access token gunicorn-registry stdout | 2025-11-04 09:33:26,307 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,308 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,311 [292] [DEBUG] [auth.credentials] Successfully validated credentials for access token 3 gunicorn-registry stdout | 2025-11-04 09:33:26,311 [292] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [None, None, , None, None, None, None]) gunicorn-registry stdout | 2025-11-04 09:33:26,311 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,311 [292] [DEBUG] [auth.permissions] Loading permissions for token: K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X gunicorn-registry stdout | 2025-11-04 09:33:26,312 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,312 [299] [DEBUG] [auth.credentials] Successfully validated credentials for access token 3 gunicorn-registry stdout | 2025-11-04 09:33:26,312 [299] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [None, None, , None, None, None, None]) gunicorn-registry stdout | 2025-11-04 09:33:26,313 [299] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,313 [299] [DEBUG] [auth.permissions] Loading permissions for token: K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X gunicorn-registry stdout | 2025-11-04 09:33:26,313 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,316 [292] [DEBUG] [auth.permissions] Delegate token added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='write') gunicorn-registry stdout | 2025-11-04 09:33:26,316 [292] [DEBUG] [endpoints.v2.v2auth] Request audience: quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com gunicorn-registry stdout | 2025-11-04 09:33:26,316 [292] [DEBUG] [endpoints.v2.v2auth] Scope request: ['repository:build_test/upload_file_firefox:pull,push'] gunicorn-registry stdout | 2025-11-04 09:33:26,316 [292] [DEBUG] [endpoints.v2.v2auth] Match: ('build_test/upload_file_firefox', 'build_test/upload_file_firefox', 'pull,push') gunicorn-registry stdout | 2025-11-04 09:33:26,316 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,317 [299] [DEBUG] [auth.permissions] Delegate token added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='write') gunicorn-registry stdout | 2025-11-04 09:33:26,317 [299] [DEBUG] [endpoints.v2.v2auth] Request audience: quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com gunicorn-registry stdout | 2025-11-04 09:33:26,317 [299] [DEBUG] [endpoints.v2.v2auth] Scope request: ['repository:build_test/upload_file_firefox:pull,push'] gunicorn-registry stdout | 2025-11-04 09:33:26,317 [299] [DEBUG] [endpoints.v2.v2auth] Match: ('build_test/upload_file_firefox', 'build_test/upload_file_firefox', 'pull,push') gunicorn-registry stdout | 2025-11-04 09:33:26,318 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,320 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,321 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,323 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,325 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,326 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,328 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,330 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,331 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,375 [292] [DEBUG] [app] Ending request: urn:request:0adcf829-f873-44c0-a3d3-55f04dffa39e (/v2/auth) {'endpoint': 'v2.generate_registry_jwt', 'request_id': 'urn:request:0adcf829-f873-44c0-a3d3-55f04dffa39e', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/auth?account=$token&scope=repository:build_test/upload_file_firefox:pull,push&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com', 'path': '/v2/auth', 'parameters': {'account': '$token', 'scope': 'repository:build_test/upload_file_firefox:pull,push', 'service': 'quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com'}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,375 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,375 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,375 [292] [INFO] [gunicorn.access] 10.131.0.6 - $token [04/Nov/2025:09:33:26 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull%2Cpush&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1251 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - $token [04/Nov/2025:09:33:26 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull%2Cpush&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1251 "-" "containers/5.29.5 (github.com/containers/image)" (0.070 494 0.070) gunicorn-registry stdout | 2025-11-04 09:33:26,376 [299] [DEBUG] [app] Ending request: urn:request:9de5939c-33fc-45cf-9804-7028339e5828 (/v2/auth) {'endpoint': 'v2.generate_registry_jwt', 'request_id': 'urn:request:9de5939c-33fc-45cf-9804-7028339e5828', 'remote_addr': '10.129.2.10', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/auth?account=$token&scope=repository:build_test/upload_file_firefox:pull,push&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com', 'path': '/v2/auth', 'parameters': {'account': '$token', 'scope': 'repository:build_test/upload_file_firefox:pull,push', 'service': 'quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com'}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,376 [299] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,376 [299] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,377 [299] [INFO] [gunicorn.access] 10.129.2.10 - $token [04/Nov/2025:09:33:26 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull%2Cpush&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1251 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - $token [04/Nov/2025:09:33:26 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull%2Cpush&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1251 "-" "containers/5.29.5 (github.com/containers/image)" (0.071 494 0.071) gunicorn-registry stdout | 2025-11-04 09:33:26,389 [299] [DEBUG] [app] Starting request: urn:request:81adeeac-4039-44a1-a894-a3e4c06135d3 (/v2/build_test/upload_file_firefox/blobs/sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:26,389 [299] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'digest': 'sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:26,389 [299] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,389 [299] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,390 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 33, 26, 389446), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 33, 26, 389465), 'quay', None, datetime.datetime(2025, 11, 3, 9, 33, 26, 389479), datetime.datetime(2025, 10, 28, 9, 33, 26, 389525), None]) gunicorn-registry stdout | 2025-11-04 09:33:26,390 [298] [DEBUG] [app] Starting request: urn:request:2225c11c-0b7f-47b2-b8d6-9e3434a6dbbc (/v2/build_test/upload_file_firefox/blobs/sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:26,390 [298] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'digest': 'sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:26,390 [298] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,391 [298] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,393 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 33, 26, 392077), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 33, 26, 392105), 'quay', None, datetime.datetime(2025, 11, 3, 9, 33, 26, 392130), datetime.datetime(2025, 10, 28, 9, 33, 26, 392146), None]) gunicorn-registry stdout | 2025-11-04 09:33:26,393 [298] [DEBUG] [peewee.pool] No connection available in pool. gunicorn-registry stdout | 2025-11-04 09:33:26,394 [299] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,394 [299] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:26,394 [299] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:26,394 [299] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:26,394 [299] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:26,395 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,399 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,400 [298] [DEBUG] [peewee.pool] Created new connection 140083010304448. gunicorn-registry stdout | 2025-11-04 09:33:26,402 [299] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['build_test', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:33:26,402 [299] [DEBUG] [data.cache.impl] Checking cache for key repo_blob__build_test_upload_file_firefox_sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe_2 gunicorn-registry stdout | 2025-11-04 09:33:26,402 [299] [DEBUG] [data.cache.impl] Found no result in cache for key repo_blob__build_test_upload_file_firefox_sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe_2; calling loader gunicorn-registry stdout | 2025-11-04 09:33:26,403 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,406 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,407 [298] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,407 [298] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:26,407 [298] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:26,407 [298] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:26,407 [298] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:26,409 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,412 [299] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,414 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,415 [299] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,419 [299] [DEBUG] [data.cache.impl] Got loaded result for key repo_blob__build_test_upload_file_firefox_sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe_2: None gunicorn-registry stdout | 2025-11-04 09:33:26,419 [299] [DEBUG] [data.cache.impl] Not caching loaded result for key repo_blob__build_test_upload_file_firefox_sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe_2: None gunicorn-registry stdout | 2025-11-04 09:33:26,419 [299] [DEBUG] [endpoints.v2] sending response: b'{"errors":[{"code":"BLOB_UNKNOWN","detail":{},"message":"blob unknown to registry"}]}\n' gunicorn-registry stdout | 2025-11-04 09:33:26,419 [299] [DEBUG] [app] Ending request: urn:request:81adeeac-4039-44a1-a894-a3e4c06135d3 (/v2/build_test/upload_file_firefox/blobs/sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe) {'endpoint': 'v2.check_blob_exists', 'request_id': 'urn:request:81adeeac-4039-44a1-a894-a3e4c06135d3', 'remote_addr': '10.129.2.10', 'http_method': 'HEAD', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe', 'path': '/v2/build_test/upload_file_firefox/blobs/sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,419 [299] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,419 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,419 [299] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,419 [299] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:26 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe HTTP/1.1" 404 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:26 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:be1f7344519a2b9b41aa7154581b1b5503c2168abb3bc1edc99b656dca2426fe HTTP/1.1" 404 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.031 1591 0.032) gunicorn-registry stdout | 2025-11-04 09:33:26,422 [298] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['build_test', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:33:26,423 [298] [DEBUG] [data.cache.impl] Checking cache for key repo_blob__build_test_upload_file_firefox_sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d_2 gunicorn-registry stdout | 2025-11-04 09:33:26,423 [298] [DEBUG] [data.cache.impl] Found no result in cache for key repo_blob__build_test_upload_file_firefox_sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d_2; calling loader gunicorn-registry stdout | 2025-11-04 09:33:26,424 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,427 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,430 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,431 [299] [DEBUG] [app] Starting request: urn:request:d51fc679-78e9-4556-88d3-105f9abd6076 (/v2/auth) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:26,431 [299] [DEBUG] [auth.basic] Attempt to process basic auth header gunicorn-registry stdout | 2025-11-04 09:33:26,431 [299] [DEBUG] [auth.credentials] Found credentials for access token gunicorn-registry stdout | 2025-11-04 09:33:26,432 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,434 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:33:26,436 [299] [DEBUG] [auth.credentials] Successfully validated credentials for access token 3 gunicorn-registry stdout | 2025-11-04 09:33:26,436 [299] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [None, None, , None, None, None, None]) gunicorn-registry stdout | 2025-11-04 09:33:26,436 [299] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,436 [299] [DEBUG] [auth.permissions] Loading permissions for token: K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X gunicorn-registry stdout | 2025-11-04 09:33:26,437 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,437 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['public', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,440 [299] [DEBUG] [auth.permissions] Delegate token added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='write') gunicorn-registry stdout | 2025-11-04 09:33:26,440 [299] [DEBUG] [endpoints.v2.v2auth] Request audience: quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com gunicorn-registry stdout | 2025-11-04 09:33:26,440 [299] [DEBUG] [endpoints.v2.v2auth] Scope request: ['repository:build_test/upload_file_firefox:pull,push', 'repository:build_test/upload_file_firefox:pull'] gunicorn-registry stdout | 2025-11-04 09:33:26,440 [299] [DEBUG] [endpoints.v2.v2auth] Match: ('build_test/upload_file_firefox', 'build_test/upload_file_firefox', 'pull,push') gunicorn-registry stdout | 2025-11-04 09:33:26,440 [298] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,441 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,444 [298] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,444 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,447 [298] [DEBUG] [data.cache.impl] Got loaded result for key repo_blob__build_test_upload_file_firefox_sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d_2: None gunicorn-registry stdout | 2025-11-04 09:33:26,447 [298] [DEBUG] [data.cache.impl] Not caching loaded result for key repo_blob__build_test_upload_file_firefox_sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d_2: None gunicorn-registry stdout | 2025-11-04 09:33:26,447 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,447 [298] [DEBUG] [endpoints.v2] sending response: b'{"errors":[{"code":"BLOB_UNKNOWN","detail":{},"message":"blob unknown to registry"}]}\n' gunicorn-registry stdout | 2025-11-04 09:33:26,448 [298] [DEBUG] [app] Ending request: urn:request:2225c11c-0b7f-47b2-b8d6-9e3434a6dbbc (/v2/build_test/upload_file_firefox/blobs/sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d) {'endpoint': 'v2.check_blob_exists', 'request_id': 'urn:request:2225c11c-0b7f-47b2-b8d6-9e3434a6dbbc', 'remote_addr': '10.131.0.6', 'http_method': 'HEAD', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d', 'path': '/v2/build_test/upload_file_firefox/blobs/sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,448 [298] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,448 [298] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,449 [298] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:26 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d HTTP/1.1" 404 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:26 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:c1f9d285a5d247a5303d7f7acea7e20a64f1a9012612ffbfdadb0397bb8a185d HTTP/1.1" 404 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.061 1591 0.060) gunicorn-registry stdout | 2025-11-04 09:33:26,450 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,454 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,457 [299] [DEBUG] [endpoints.v2.v2auth] Match: ('build_test/upload_file_firefox', 'build_test/upload_file_firefox', 'pull') gunicorn-registry stdout | 2025-11-04 09:33:26,457 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,461 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,464 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,467 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,471 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,516 [299] [DEBUG] [app] Ending request: urn:request:d51fc679-78e9-4556-88d3-105f9abd6076 (/v2/auth) {'endpoint': 'v2.generate_registry_jwt', 'request_id': 'urn:request:d51fc679-78e9-4556-88d3-105f9abd6076', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/auth?account=$token&scope=repository:build_test/upload_file_firefox:pull,push&scope=repository:build_test/upload_file_firefox:pull&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com', 'path': '/v2/auth', 'parameters': {'account': '$token', 'scope': 'repository:build_test/upload_file_firefox:pull,push', 'service': 'quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com'}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,516 [299] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,516 [299] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,516 [299] [INFO] [gunicorn.access] 10.131.0.6 - $token [04/Nov/2025:09:33:26 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull%2Cpush&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1359 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - $token [04/Nov/2025:09:33:26 +0000] "GET /v2/auth?account=%24token&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull%2Cpush&scope=repository%3Abuild_test%2Fupload_file_firefox%3Apull&service=quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com HTTP/1.1" 200 1359 "-" "containers/5.29.5 (github.com/containers/image)" (0.086 553 0.085) gunicorn-registry stdout | 2025-11-04 09:33:26,527 [299] [DEBUG] [app] Starting request: urn:request:5dff91ac-e71a-4006-9e16-5d495cf23f9d (/v2/build_test/upload_file_firefox/blobs/sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:26,528 [299] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'digest': 'sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:26,528 [299] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfSx7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.VaS3Bz622HG1FGMeZNQXjoDLDGPO4mMvsAaA1ISAgsjezwta0AW1YfuZTXSofpihMBubrLhKRkPPC2y3LJYDev__UCph_BN0mU2wrChAeXs7ZPs1w2gzmCSrgtKFj_WPhOgZxmURIZVrjcO6pF6Wn-4ruL8fiI4AKWWaXFIUhTtw3WVqiqvaORFyzj2Jj2Lb3X43XD6Qvwob2ytbMKFRdhNzpU7hH7soIbRDJ0tSI3fNgU3ns4WqTKXQHlbXmEO14Cl-BaYKLBBnjGZN1IIXauNZCRsIcYObCiutwr1m9aYLc-EP37jRzL3_WI8NUG_FWbvR8E35IYolscIRr3owCQ gunicorn-registry stdout | 2025-11-04 09:33:26,528 [299] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfSx7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6ImJ1aWxkX3Rlc3QvdXBsb2FkX2ZpbGVfZmlyZWZveCIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.VaS3Bz622HG1FGMeZNQXjoDLDGPO4mMvsAaA1ISAgsjezwta0AW1YfuZTXSofpihMBubrLhKRkPPC2y3LJYDev__UCph_BN0mU2wrChAeXs7ZPs1w2gzmCSrgtKFj_WPhOgZxmURIZVrjcO6pF6Wn-4ruL8fiI4AKWWaXFIUhTtw3WVqiqvaORFyzj2Jj2Lb3X43XD6Qvwob2ytbMKFRdhNzpU7hH7soIbRDJ0tSI3fNgU3ns4WqTKXQHlbXmEO14Cl-BaYKLBBnjGZN1IIXauNZCRsIcYObCiutwr1m9aYLc-EP37jRzL3_WI8NUG_FWbvR8E35IYolscIRr3owCQ gunicorn-registry stdout | 2025-11-04 09:33:26,528 [299] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,528 [299] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:26,529 [299] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:26,529 [299] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:26,529 [299] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:26,529 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,533 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,537 [299] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['build_test', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:33:26,537 [299] [DEBUG] [data.cache.impl] Checking cache for key repo_blob__build_test_upload_file_firefox_sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5_2 gunicorn-registry stdout | 2025-11-04 09:33:26,537 [299] [DEBUG] [data.cache.impl] Found no result in cache for key repo_blob__build_test_upload_file_firefox_sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5_2; calling loader gunicorn-registry stdout | 2025-11-04 09:33:26,538 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,541 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,544 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,547 [299] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,551 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['4331528b-8d4d-470e-a25d-cf79b7c8c31e']) gunicorn-registry stdout | 2025-11-04 09:33:26,552 [298] [DEBUG] [app] Starting request: urn:request:a264a2ad-3525-4e81-8483-5b7442731a66 (/v2/build_test/upload_file_firefox/blobs/uploads/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:26,552 [298] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:26,552 [298] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,552 [298] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,553 [298] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,553 [298] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:26,553 [298] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:26,553 [298] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:26,553 [298] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:26,554 [299] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['4331528b-8d4d-470e-a25d-cf79b7c8c31e']) gunicorn-registry stdout | 2025-11-04 09:33:26,554 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,557 [299] [DEBUG] [data.cache.impl] Got loaded result for key repo_blob__build_test_upload_file_firefox_sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5_2: {'uuid': '4331528b-8d4d-470e-a25d-cf79b7c8c31e', 'digest': 'sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 'compressed_size': 80005019, 'uncompressed_size': 218549248, 'uploading': True, 'db_id': 2, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/7f/7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5'}} gunicorn-registry stdout | 2025-11-04 09:33:26,557 [299] [DEBUG] [data.cache.impl] Caching loaded result for key repo_blob__build_test_upload_file_firefox_sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5_2 with expiration {'uuid': '4331528b-8d4d-470e-a25d-cf79b7c8c31e', 'digest': 'sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 'compressed_size': 80005019, 'uncompressed_size': 218549248, 'uploading': True, 'db_id': 2, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/7f/7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5'}}: 60s gunicorn-registry stdout | 2025-11-04 09:33:26,557 [299] [DEBUG] [data.cache.impl] Cached loaded result for key repo_blob__build_test_upload_file_firefox_sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5_2 with expiration {'uuid': '4331528b-8d4d-470e-a25d-cf79b7c8c31e', 'digest': 'sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 'compressed_size': 80005019, 'uncompressed_size': 218549248, 'uploading': True, 'db_id': 2, 'inputs': {'placements': ['default'], 'storage_path': 'sha256/7f/7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5'}}: 60s gunicorn-registry stdout | 2025-11-04 09:33:26,557 [299] [DEBUG] [app] Ending request: urn:request:5dff91ac-e71a-4006-9e16-5d495cf23f9d (/v2/build_test/upload_file_firefox/blobs/sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5) {'endpoint': 'v2.check_blob_exists', 'request_id': 'urn:request:5dff91ac-e71a-4006-9e16-5d495cf23f9d', 'remote_addr': '10.131.0.6', 'http_method': 'HEAD', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 'path': '/v2/build_test/upload_file_firefox/blobs/sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,557 [299] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,557 [299] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,558 [299] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:26 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5 HTTP/1.1" 200 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:26 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5 HTTP/1.1" 200 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.031 1699 0.031) gunicorn-registry stdout | 2025-11-04 09:33:26,558 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,561 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,565 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,568 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,572 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,575 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:26,579 [298] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/endpoints.json gunicorn-registry stdout | 2025-11-04 09:33:26,591 [298] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/sdk-default-configuration.json gunicorn-registry stdout | 2025-11-04 09:33:26,591 [298] [DEBUG] [botocore.hooks] Event choose-service-name: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,607 [298] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/s3/2006-03-01/service-2.json gunicorn-registry stdout | 2025-11-04 09:33:26,628 [298] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/s3/2006-03-01/endpoint-rule-set-1.json gunicorn-registry stdout | 2025-11-04 09:33:26,630 [298] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/partitions.json gunicorn-registry stdout | 2025-11-04 09:33:26,632 [298] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,632 [298] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler ._handler at 0x7f679f49da80> gunicorn-registry stdout | 2025-11-04 09:33:26,651 [298] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,653 [298] [DEBUG] [botocore.endpoint] Setting s3 timeout as (60, 60) gunicorn-registry stdout | 2025-11-04 09:33:26,655 [298] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/_retry.json gunicorn-registry stdout | 2025-11-04 09:33:26,656 [298] [DEBUG] [botocore.client] Registering retry handlers for service: s3 gunicorn-registry stdout | 2025-11-04 09:33:26,656 [298] [DEBUG] [botocore.utils] Registering S3 region redirector handler gunicorn-registry stdout | 2025-11-04 09:33:26,674 [298] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/boto3/data/s3/2006-03-01/resources-1.json gunicorn-registry stdout | 2025-11-04 09:33:26,675 [298] [DEBUG] [botocore.hooks] Event choose-service-name: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,676 [298] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,676 [298] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler ._handler at 0x7f679f49da80> gunicorn-registry stdout | 2025-11-04 09:33:26,676 [298] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,676 [298] [DEBUG] [botocore.endpoint] Setting s3 timeout as (60, 60) gunicorn-registry stdout | 2025-11-04 09:33:26,678 [298] [DEBUG] [botocore.client] Registering retry handlers for service: s3 gunicorn-registry stdout | 2025-11-04 09:33:26,678 [298] [DEBUG] [botocore.utils] Registering S3 region redirector handler gunicorn-registry stdout | 2025-11-04 09:33:26,678 [298] [DEBUG] [boto3.resources.factory] Loading s3:s3 gunicorn-registry stdout | 2025-11-04 09:33:26,679 [298] [DEBUG] [boto3.resources.factory] Loading s3:Bucket gunicorn-registry stdout | 2025-11-04 09:33:26,679 [298] [DEBUG] [boto3.resources.model] Renaming Bucket attribute name gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Bucket: calling handler ._handler at 0x7f679f49dda0> gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,680 [298] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:26,681 [298] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:26,681 [298] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:26,681 [298] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadBucket) with params: {'url_path': '', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.hooks] Event request-created.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,682 [298] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:26,683 [298] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | HEAD gunicorn-registry stdout | /whuaws gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093326Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:26,683 [298] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093326Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | b03fd9c90f882c0d0c9ada5179fadba3c4fb198074a977326ff5c2a5e434d7d2 gunicorn-registry stdout | 2025-11-04 09:33:26,683 [298] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | e2bbb30754768298d27c727de10388c831772679e2641a8defa6a1e886b4274a gunicorn-registry stdout | 2025-11-04 09:33:26,683 [298] [DEBUG] [botocore.hooks] Event request-created.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,683 [298] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:26,683 [298] [DEBUG] [botocore.hooks] Event before-send.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,683 [298] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093326Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e2bbb30754768298d27c727de10388c831772679e2641a8defa6a1e886b4274a', 'amz-sdk-invocation-id': b'e77eb347-9ede-4891-aaca-2247187efe1f', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:26,684 [298] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:26,684 [298] [DEBUG] [urllib3.connectionpool] Starting new HTTPS connection (1): s3.us-east-1.amazonaws.com:443 gunicorn-registry stdout | 2025-11-04 09:33:26,767 [298] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "HEAD /whuaws HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:26,767 [298] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': '1+rR+R5bklN7wdEelEBpeXzPtc4pyl1BFGV/X6f/VdKEFVIp/ZIMR5nzV4EX2VnkCDFBPdy9p2E=', 'x-amz-request-id': 'XMMWZY8A56FK27HK', 'Date': 'Tue, 04 Nov 2025 09:33:27 GMT', 'x-amz-bucket-region': 'us-east-1', 'x-amz-access-point-alias': 'false', 'x-amz-bucket-arn': 'arn:aws:s3:::whuaws', 'Content-Type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:26,767 [298] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:26,768 [298] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,768 [298] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:26,768 [298] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,768 [298] [DEBUG] [botocore.hooks] Event after-call.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,768 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,772 [298] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "imagestoragelocation" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:33:26,776 [298] [DEBUG] [peewee] ('INSERT INTO "blobupload" ("repository_id", "uuid", "byte_count", "sha_state", "location_id", "storage_metadata", "chunk_count", "created") VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING "blobupload"."id"', [2, '6256983b-3cf4-4041-9d44-3b3f77944f02', 0, 'gASVlQAAAAAAAACMD3Jlc3VtYWJsZXNoYTI1NpSMBnNoYTI1NpSTlCmBlENwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW5RiLg==', 9, '{"chunks": []}', 0, datetime.datetime(2025, 11, 4, 9, 33, 26, 775929)]) gunicorn-registry stdout | 2025-11-04 09:33:26,781 [298] [DEBUG] [app] Ending request: urn:request:a264a2ad-3525-4e81-8483-5b7442731a66 (/v2/build_test/upload_file_firefox/blobs/uploads/) {'endpoint': 'v2.start_blob_upload', 'request_id': 'urn:request:a264a2ad-3525-4e81-8483-5b7442731a66', 'remote_addr': '10.129.2.10', 'http_method': 'POST', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/uploads/', 'path': '/v2/build_test/upload_file_firefox/blobs/uploads/', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:26,782 [298] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,782 [298] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,782 [298] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:26 +0000] "POST /v2/build_test/upload_file_firefox/blobs/uploads/ HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:26 +0000] "POST /v2/build_test/upload_file_firefox/blobs/uploads/ HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.231 1570 0.231) gunicorn-registry stdout | 2025-11-04 09:33:26,803 [292] [DEBUG] [app] Starting request: urn:request:f16b264c-92d5-49f9-9a71-b070b16b94fd (/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:26,803 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'upload_uuid': '6256983b-3cf4-4041-9d44-3b3f77944f02', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:26,803 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,803 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:26,804 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:26,804 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:26,804 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:26,804 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:26,804 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:26,805 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,809 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,812 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,815 [292] [DEBUG] [peewee] ('SELECT Sum("t1"."byte_count") AS "size_bytes" FROM "blobupload" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,819 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,822 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,825 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,828 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:26,831 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['6256983b-3cf4-4041-9d44-3b3f77944f02', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:26,835 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:26,835 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:26,835 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:26,836 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [boto3.resources.action] Calling s3:create_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'} gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,837 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CreateMultipartUpload) with params: {'url_path': '/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploads', 'query_string': {}, 'method': 'POST', 'headers': {'Content-Type': 'application/octet-stream', 'x-amz-server-side-encryption': 'AES256', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploads', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | uploads= gunicorn-registry stdout | content-type:application/octet-stream gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093326Z gunicorn-registry stdout | x-amz-server-side-encryption:AES256 gunicorn-registry stdout | content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093326Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 8ed369d84810d8d218fbd80cddea9620de5c0d7024ce960bf89f93780428819b gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | de4063c80dcad3d0b3281302b01bd4105f368ab9987e01500d41b44d8d9da71e gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,838 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Type': b'application/octet-stream', 'x-amz-server-side-encryption': b'AES256', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093326Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption, Signature=de4063c80dcad3d0b3281302b01bd4105f368ab9987e01500d41b44d8d9da71e', 'amz-sdk-invocation-id': b'ef121f30-02a0-4fab-9f47-91592a18ee3b', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:26,839 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:26,839 [292] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: s3.us-east-1.amazonaws.com gunicorn-registry stdout | 2025-11-04 09:33:26,903 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploads HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:26,904 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'sbOe9cfkixYbeWQdHgOdr0SY+Eu3p8gNtHKj11RzzgFL8Z3Xv6m3aY6u6MRaT5eWiwMvhq70RGA=', 'x-amz-request-id': 'XMMHSQR1KRSSFERE', 'Date': 'Tue, 04 Nov 2025 09:33:27 GMT', 'x-amz-server-side-encryption': 'AES256', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:26,904 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\nwhuawsquaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc248S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8' gunicorn-registry stdout | 2025-11-04 09:33:26,904 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:26,904 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:26,904 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,905 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:26,905 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'XMMHSQR1KRSSFERE', 'HostId': 'sbOe9cfkixYbeWQdHgOdr0SY+Eu3p8gNtHKj11RzzgFL8Z3Xv6m3aY6u6MRaT5eWiwMvhq70RGA=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'sbOe9cfkixYbeWQdHgOdr0SY+Eu3p8gNtHKj11RzzgFL8Z3Xv6m3aY6u6MRaT5eWiwMvhq70RGA=', 'x-amz-request-id': 'XMMHSQR1KRSSFERE', 'date': 'Tue, 04 Nov 2025 09:33:27 GMT', 'x-amz-server-side-encryption': 'AES256', 'transfer-encoding': 'chunked', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8'} gunicorn-registry stdout | 2025-11-04 09:33:26,905 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUpload gunicorn-registry stdout | 2025-11-04 09:33:27,025 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUploadPart gunicorn-registry stdout | 2025-11-04 09:33:27,025 [292] [DEBUG] [boto3.resources.model] Renaming MultipartUploadPart attribute part_number gunicorn-registry stdout | 2025-11-04 09:33:27,025 [292] [DEBUG] [boto3.resources.action] Calling s3:upload_part with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 1, 'Body': <_io.BytesIO object at 0x7f679cb93f60>, 'ContentLength': 5242880} gunicorn-registry stdout | 2025-11-04 09:33:27,025 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:27,026 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,034 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,034 [292] [DEBUG] [botocore.handlers] Adding expect 100 continue header to request. gunicorn-registry stdout | 2025-11-04 09:33:27,034 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,034 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,034 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPart) with params: {'url_path': '/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'query_string': {'uploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'partNumber': 1}, 'method': 'PUT', 'headers': {'Content-Length': '5242880', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': '0E3F8JCI/rR2wej/+wwfXA==', 'Expect': '100-continue'}, 'body': <_io.BytesIO object at 0x7f679cb93f60>, 'auth_path': '/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=1', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': True, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 1, 'Body': <_io.BytesIO object at 0x7f679cb93f60>, 'ContentLength': 5242880}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:27,034 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | partNumber=1&uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8 gunicorn-registry stdout | content-length:5242880 gunicorn-registry stdout | content-md5:0E3F8JCI/rR2wej/+wwfXA== gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:UNSIGNED-PAYLOAD gunicorn-registry stdout | x-amz-date:20251104T093327Z gunicorn-registry stdout | content-length;content-md5;host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | UNSIGNED-PAYLOAD gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093327Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 32402d4a3eb033a8a8cc46c732685db054ff91584ce8d1539049d11745fafef5 gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 7f8359f9c48b847be60a8a3ddb61173d44dbe6182ce9de18dad42c3ef461710b gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Length': b'5242880', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': b'0E3F8JCI/rR2wej/+wwfXA==', 'Expect': b'100-continue', 'X-Amz-Date': b'20251104T093327Z', 'X-Amz-Content-SHA256': b'UNSIGNED-PAYLOAD', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date, Signature=7f8359f9c48b847be60a8a3ddb61173d44dbe6182ce9de18dad42c3ef461710b', 'amz-sdk-invocation-id': b'832cbedc-be14-400c-abf2-3c74bbb40151', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:27,035 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:27,036 [292] [DEBUG] [botocore.awsrequest] Waiting for 100 Continue response. gunicorn-registry stdout | 2025-11-04 09:33:27,067 [292] [DEBUG] [botocore.awsrequest] 100 Continue response seen, now sending request body. proxycacheblobworker stdout | 2025-11-04 09:33:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:33:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:37.122849+00:00 (in 9.999635 seconds) proxycacheblobworker stdout | 2025-11-04 09:33:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:37 GMT)" (scheduled at 2025-11-04 09:33:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:33:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:33:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 27, 123479), True, datetime.datetime(2025, 11, 4, 9, 33, 27, 123479), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:33:27,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:33:27,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:33:27,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:37 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:33:27,166 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=1 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:27,167 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'CQbL9n/nZ1leYTk20c3YBeUEJWLUcr+AcTqxOrEUUrIrvIjacZOEJukTEDBwA4hqHU6t/+XiTv4=', 'x-amz-request-id': 'EFRGZJ4KZM1SEF5P', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'ETag': '"d04dc5f09088feb476c1e8fffb0c1f5c"', 'x-amz-server-side-encryption': 'AES256', 'Content-Length': '0', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:27,167 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:27,167 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,167 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:27,167 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,167 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,167 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'EFRGZJ4KZM1SEF5P', 'HostId': 'CQbL9n/nZ1leYTk20c3YBeUEJWLUcr+AcTqxOrEUUrIrvIjacZOEJukTEDBwA4hqHU6t/+XiTv4=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'CQbL9n/nZ1leYTk20c3YBeUEJWLUcr+AcTqxOrEUUrIrvIjacZOEJukTEDBwA4hqHU6t/+XiTv4=', 'x-amz-request-id': 'EFRGZJ4KZM1SEF5P', 'date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'etag': '"d04dc5f09088feb476c1e8fffb0c1f5c"', 'x-amz-server-side-encryption': 'AES256', 'content-length': '0', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'ETag': '"d04dc5f09088feb476c1e8fffb0c1f5c"'} buildlogsarchiver stdout | 2025-11-04 09:33:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:33:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:57.222846+00:00 (in 29.999606 seconds) buildlogsarchiver stdout | 2025-11-04 09:33:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:57 GMT)" (scheduled at 2025-11-04 09:33:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:33:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 33, 27, 223447), False, 50, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,224 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUploadPart gunicorn-registry stdout | 2025-11-04 09:33:27,224 [292] [DEBUG] [boto3.resources.model] Renaming MultipartUploadPart attribute part_number gunicorn-registry stdout | 2025-11-04 09:33:27,224 [292] [DEBUG] [boto3.resources.action] Calling s3:upload_part with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 2, 'Body': <_io.BytesIO object at 0x7f679cb4c9a0>, 'ContentLength': 5242880} gunicorn-registry stdout | 2025-11-04 09:33:27,224 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,224 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,224 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,224 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:27,225 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,233 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,233 [292] [DEBUG] [botocore.handlers] Adding expect 100 continue header to request. gunicorn-registry stdout | 2025-11-04 09:33:27,233 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPart) with params: {'url_path': '/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'query_string': {'uploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'partNumber': 2}, 'method': 'PUT', 'headers': {'Content-Length': '5242880', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': 'syq7V/sfqwO/p5HRbKCw8Q==', 'Expect': '100-continue'}, 'body': <_io.BytesIO object at 0x7f679cb4c9a0>, 'auth_path': '/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': True, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 2, 'Body': <_io.BytesIO object at 0x7f679cb4c9a0>, 'ContentLength': 5242880}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | partNumber=2&uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8 gunicorn-registry stdout | content-length:5242880 gunicorn-registry stdout | content-md5:syq7V/sfqwO/p5HRbKCw8Q== gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:UNSIGNED-PAYLOAD gunicorn-registry stdout | x-amz-date:20251104T093327Z gunicorn-registry stdout | content-length;content-md5;host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | UNSIGNED-PAYLOAD gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093327Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 1a821a82706d7aecb017913cd5ef981cab970b9759aa7a1de4c260e03b67c85e gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 81360980e8ee5bba8c982b655af887a0a0b0ed616b20b7e3a2287c1ab77bcfc0 gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,234 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Length': b'5242880', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': b'syq7V/sfqwO/p5HRbKCw8Q==', 'Expect': b'100-continue', 'X-Amz-Date': b'20251104T093327Z', 'X-Amz-Content-SHA256': b'UNSIGNED-PAYLOAD', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date, Signature=81360980e8ee5bba8c982b655af887a0a0b0ed616b20b7e3a2287c1ab77bcfc0', 'amz-sdk-invocation-id': b'776dec71-5289-4d1c-b08d-7e78db8f2cb1', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:27,235 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:27,235 [292] [DEBUG] [botocore.awsrequest] Waiting for 100 Continue response. buildlogsarchiver stdout | 2025-11-04 09:33:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:33:27,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:33:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:33:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:33:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:33:57 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:33:27,274 [292] [DEBUG] [botocore.awsrequest] 100 Continue response seen, now sending request body. gunicorn-registry stdout | 2025-11-04 09:33:27,417 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=2 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:27,417 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'MLSUE9j5hKBiS+Is3PmHkw60Mq8PGJG2e/hXpJpq+L37gl4yOqAwUsfME7e7NgUBZQVGaEIrbUM=', 'x-amz-request-id': 'EFRQ57A98CXEPERN', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'ETag': '"b32abb57fb1fab03bfa791d16ca0b0f1"', 'x-amz-server-side-encryption': 'AES256', 'Content-Length': '0', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:27,417 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:27,417 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,417 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:27,417 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,417 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,418 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'EFRQ57A98CXEPERN', 'HostId': 'MLSUE9j5hKBiS+Is3PmHkw60Mq8PGJG2e/hXpJpq+L37gl4yOqAwUsfME7e7NgUBZQVGaEIrbUM=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'MLSUE9j5hKBiS+Is3PmHkw60Mq8PGJG2e/hXpJpq+L37gl4yOqAwUsfME7e7NgUBZQVGaEIrbUM=', 'x-amz-request-id': 'EFRQ57A98CXEPERN', 'date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'etag': '"b32abb57fb1fab03bfa791d16ca0b0f1"', 'x-amz-server-side-encryption': 'AES256', 'content-length': '0', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'ETag': '"b32abb57fb1fab03bfa791d16ca0b0f1"'} gunicorn-registry stdout | 2025-11-04 09:33:27,477 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUploadPart gunicorn-registry stdout | 2025-11-04 09:33:27,477 [292] [DEBUG] [boto3.resources.model] Renaming MultipartUploadPart attribute part_number gunicorn-registry stdout | 2025-11-04 09:33:27,477 [292] [DEBUG] [boto3.resources.action] Calling s3:upload_part with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 3, 'Body': <_io.BytesIO object at 0x7f67a1ab62f0>, 'ContentLength': 5242880} gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:27,478 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.handlers] Adding expect 100 continue header to request. gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPart) with params: {'url_path': '/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'query_string': {'uploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'partNumber': 3}, 'method': 'PUT', 'headers': {'Content-Length': '5242880', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': '3NttYoUJbVFtJuysHkRgHQ==', 'Expect': '100-continue'}, 'body': <_io.BytesIO object at 0x7f67a1ab62f0>, 'auth_path': '/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=3', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': True, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 3, 'Body': <_io.BytesIO object at 0x7f67a1ab62f0>, 'ContentLength': 5242880}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | partNumber=3&uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8 gunicorn-registry stdout | content-length:5242880 gunicorn-registry stdout | content-md5:3NttYoUJbVFtJuysHkRgHQ== gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:UNSIGNED-PAYLOAD gunicorn-registry stdout | x-amz-date:20251104T093327Z gunicorn-registry stdout | content-length;content-md5;host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | UNSIGNED-PAYLOAD gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093327Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 205392fdd348f36e353a7e65fb86c5407736919bea2cf88fb78cce471c495bc9 gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | febea68b1a0a40be0062d409d4f747356b44c8d320c4bc2baed7e00d5c7f9439 gunicorn-registry stdout | 2025-11-04 09:33:27,487 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,488 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:27,488 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,488 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Length': b'5242880', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': b'3NttYoUJbVFtJuysHkRgHQ==', 'Expect': b'100-continue', 'X-Amz-Date': b'20251104T093327Z', 'X-Amz-Content-SHA256': b'UNSIGNED-PAYLOAD', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date, Signature=febea68b1a0a40be0062d409d4f747356b44c8d320c4bc2baed7e00d5c7f9439', 'amz-sdk-invocation-id': b'c083f275-f5c9-4f56-b365-b27e3552d01c', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:27,488 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:27,489 [292] [DEBUG] [botocore.awsrequest] Waiting for 100 Continue response. gunicorn-registry stdout | 2025-11-04 09:33:27,506 [292] [DEBUG] [botocore.awsrequest] 100 Continue response seen, now sending request body. quotaregistrysizeworker stdout | 2025-11-04 09:33:27,530 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:27,620 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=3 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:27,620 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'BTIks1eON6rOrhFSboDXu07Xr8L7ANv6zUjLr0wi2VM0RsBqj4CCUoxDLbR7o1ETxyytL+VjP8k=', 'x-amz-request-id': 'EFRP8N9G2E0AM8HR', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'ETag': '"dcdb6d6285096d516d26ecac1e44601d"', 'x-amz-server-side-encryption': 'AES256', 'Content-Length': '0', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:27,620 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:27,620 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,620 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:27,620 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,620 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,621 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'EFRP8N9G2E0AM8HR', 'HostId': 'BTIks1eON6rOrhFSboDXu07Xr8L7ANv6zUjLr0wi2VM0RsBqj4CCUoxDLbR7o1ETxyytL+VjP8k=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'BTIks1eON6rOrhFSboDXu07Xr8L7ANv6zUjLr0wi2VM0RsBqj4CCUoxDLbR7o1ETxyytL+VjP8k=', 'x-amz-request-id': 'EFRP8N9G2E0AM8HR', 'date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'etag': '"dcdb6d6285096d516d26ecac1e44601d"', 'x-amz-server-side-encryption': 'AES256', 'content-length': '0', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'ETag': '"dcdb6d6285096d516d26ecac1e44601d"'} gunicorn-registry stdout | 2025-11-04 09:33:27,709 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUploadPart gunicorn-registry stdout | 2025-11-04 09:33:27,709 [292] [DEBUG] [boto3.resources.model] Renaming MultipartUploadPart attribute part_number gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [boto3.resources.action] Calling s3:upload_part with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 4, 'Body': <_io.BytesIO object at 0x7f679ccb7790>, 'ContentLength': 4416163} gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:27,710 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler builder stdout | 2025-11-04 09:33:27,713 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue gunicorn-registry stdout | 2025-11-04 09:33:27,717 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,717 [292] [DEBUG] [botocore.handlers] Adding expect 100 continue header to request. gunicorn-registry stdout | 2025-11-04 09:33:27,717 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPart) with params: {'url_path': '/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'query_string': {'uploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'partNumber': 4}, 'method': 'PUT', 'headers': {'Content-Length': '4416163', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': 'CBeNTh7URe+HNPZ8M7/thQ==', 'Expect': '100-continue'}, 'body': <_io.BytesIO object at 0x7f679ccb7790>, 'auth_path': '/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=4', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': True, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'PartNumber': 4, 'Body': <_io.BytesIO object at 0x7f679ccb7790>, 'ContentLength': 4416163}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | partNumber=4&uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8 gunicorn-registry stdout | content-length:4416163 gunicorn-registry stdout | content-md5:CBeNTh7URe+HNPZ8M7/thQ== gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:UNSIGNED-PAYLOAD gunicorn-registry stdout | x-amz-date:20251104T093327Z gunicorn-registry stdout | content-length;content-md5;host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | UNSIGNED-PAYLOAD gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093327Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 87c40348eb782634d19bb02063a93af1e5a7e0a34d3f8ee4a5beca7d8b147da4 gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | e752d8f7b47cd68f2912e3e81d91557b45a9ab98286e1456c903474f45fdb652 gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,718 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Length': b'4416163', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': b'CBeNTh7URe+HNPZ8M7/thQ==', 'Expect': b'100-continue', 'X-Amz-Date': b'20251104T093327Z', 'X-Amz-Content-SHA256': b'UNSIGNED-PAYLOAD', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date, Signature=e752d8f7b47cd68f2912e3e81d91557b45a9ab98286e1456c903474f45fdb652', 'amz-sdk-invocation-id': b'c49392e0-1b2c-444a-a073-5bec9cb42be7', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:27,719 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:27,719 [292] [DEBUG] [botocore.awsrequest] Waiting for 100 Continue response. builder stdout | 2025-11-04 09:33:27,726 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:33:27,727 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:33:27,737 [66] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:27,738 [292] [DEBUG] [botocore.awsrequest] 100 Continue response seen, now sending request body. gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8&partNumber=4 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'CIholNeEFHZcUpPtooyKU5Nru8hRe/RG1x42unfORhAwe4h4SayyF0g4/puiQeNP21yLsCjolRI=', 'x-amz-request-id': 'EFRW6P4SWBJYJHRF', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'ETag': '"08178d4e1ed445ef8734f67c33bfed85"', 'x-amz-server-side-encryption': 'AES256', 'Content-Length': '0', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,781 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'EFRW6P4SWBJYJHRF', 'HostId': 'CIholNeEFHZcUpPtooyKU5Nru8hRe/RG1x42unfORhAwe4h4SayyF0g4/puiQeNP21yLsCjolRI=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'CIholNeEFHZcUpPtooyKU5Nru8hRe/RG1x42unfORhAwe4h4SayyF0g4/puiQeNP21yLsCjolRI=', 'x-amz-request-id': 'EFRW6P4SWBJYJHRF', 'date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'etag': '"08178d4e1ed445ef8734f67c33bfed85"', 'x-amz-server-side-encryption': 'AES256', 'content-length': '0', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'ETag': '"08178d4e1ed445ef8734f67c33bfed85"'} gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [boto3.resources.action] Calling s3:complete_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'MultipartUpload': {'Parts': [{'ETag': '"d04dc5f09088feb476c1e8fffb0c1f5c"', 'PartNumber': 1}, {'ETag': '"b32abb57fb1fab03bfa791d16ca0b0f1"', 'PartNumber': 2}, {'ETag': '"dcdb6d6285096d516d26ecac1e44601d"', 'PartNumber': 3}, {'ETag': '"08178d4e1ed445ef8734f67c33bfed85"', 'PartNumber': 4}]}} gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,782 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CompleteMultipartUpload) with params: {'url_path': '/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'query_string': {'uploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8'}, 'method': 'POST', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'"d04dc5f09088feb476c1e8fffb0c1f5c"1"b32abb57fb1fab03bfa791d16ca0b0f1"2"dcdb6d6285096d516d26ecac1e44601d"3"08178d4e1ed445ef8734f67c33bfed85"4', 'auth_path': '/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'UploadId': '48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8', 'MultipartUpload': {'Parts': [{'ETag': '"d04dc5f09088feb476c1e8fffb0c1f5c"', 'PartNumber': 1}, {'ETag': '"b32abb57fb1fab03bfa791d16ca0b0f1"', 'PartNumber': 2}, {'ETag': '"dcdb6d6285096d516d26ecac1e44601d"', 'PartNumber': 3}, {'ETag': '"08178d4e1ed445ef8734f67c33bfed85"', 'PartNumber': 4}]}}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:b6b28b03df36b63264d628eb299def2a7838b8905de40ffe3b2c7cb016338a61 gunicorn-registry stdout | x-amz-date:20251104T093327Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | b6b28b03df36b63264d628eb299def2a7838b8905de40ffe3b2c7cb016338a61 gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093327Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | ccec205d649893860b0e65c3ffb9837385c3b0dd381d7136f76a9e15bd1b8c01 gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | c3ccc54972450e457b98f2014d1b0c283022774871acd254860aeff8ee98a11e gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,783 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093327Z', 'X-Amz-Content-SHA256': b'b6b28b03df36b63264d628eb299def2a7838b8905de40ffe3b2c7cb016338a61', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c3ccc54972450e457b98f2014d1b0c283022774871acd254860aeff8ee98a11e', 'amz-sdk-invocation-id': b'e4dbbbec-1b72-406a-8c67-47d8bdc985f0', 'amz-sdk-request': b'attempt=1', 'Content-Length': '443', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:27,784 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:27,839 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2?uploadId=48S0ncirEi4aat7irh4dRtDJfqzEY2pwhH.zLuP3gLgEGihOMbQ60sgDpsOQMWGGNqRT7aj5.gp.aKvJzB_GgUqk.vTK7zf1e2nOMDRE3roOjOMkyBsKPaC2T86Vp1H8 HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'tugWPl+i3LBsfITBKXKe1ME0gLYt+5kGzN/Oz2+pyy9hmERV1/sodAzzB0ckCFektzSwkZTDhKg=', 'x-amz-request-id': 'EFRV33WSMJ67MA4W', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'x-amz-server-side-encryption': 'AES256', 'Content-Type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\n\nhttps://s3.us-east-1.amazonaws.com/whuaws/quaydata%2Fuploads%2F1cdaf5fd-787f-473a-bddb-ee9ba1725bc2whuawsquaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2"257795cb3db8363d60a9d07d48724faa-4"N/avDMYf5jE=FULL_OBJECT' gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'EFRV33WSMJ67MA4W', 'HostId': 'tugWPl+i3LBsfITBKXKe1ME0gLYt+5kGzN/Oz2+pyy9hmERV1/sodAzzB0ckCFektzSwkZTDhKg=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'tugWPl+i3LBsfITBKXKe1ME0gLYt+5kGzN/Oz2+pyy9hmERV1/sodAzzB0ckCFektzSwkZTDhKg=', 'x-amz-request-id': 'EFRV33WSMJ67MA4W', 'date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'x-amz-server-side-encryption': 'AES256', 'content-type': 'application/xml', 'transfer-encoding': 'chunked', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'Location': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata%2Fuploads%2F1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'ETag': '"257795cb3db8363d60a9d07d48724faa-4"'} gunicorn-registry stdout | 2025-11-04 09:33:27,902 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:27,903 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:27,903 [292] [DEBUG] [data.registry_model.blobuploader] Uploaded 20144803 bytes to blob 6256983b-3cf4-4041-9d44-3b3f77944f02 took 1.0686190128326416 seconds gunicorn-registry stdout | 2025-11-04 09:33:27,904 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['6256983b-3cf4-4041-9d44-3b3f77944f02', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,908 [292] [DEBUG] [peewee] ('UPDATE "blobupload" SET "repository_id" = %s, "uuid" = %s, "byte_count" = %s, "sha_state" = %s, "location_id" = %s, "storage_metadata" = %s, "chunk_count" = %s, "uncompressed_byte_count" = %s, "created" = %s, "piece_sha_state" = %s, "piece_hashes" = %s WHERE ("blobupload"."id" = %s)', [2, '6256983b-3cf4-4041-9d44-3b3f77944f02', 20144803, 'gASVlQAAAAAAAACMD3Jlc3VtYWJsZXNoYTI1NpSMBnNoYTI1NpSTlCmBlENwCP+/X9fTpHF1Q3D4q3B4AAB44j8AAAD//wMAfk7Q3wD87AEcN30AZ87DdJ0135v+UyfnVf+n/vqP3zBiymj4fyMAAAAAAAAAABSbCQAAAACTVTzW/AOOZemeBs+hla001Ujx2TncHCfW27Z0gtNbAJRiLg==', 9, '{"chunks": [["uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2", 0, 20144803]]}', 1, 32308224, datetime.datetime(2025, 11, 4, 9, 33, 26, 775929), None, None, 4]) gunicorn-registry stdout | 2025-11-04 09:33:27,913 [292] [DEBUG] [app] Ending request: urn:request:f16b264c-92d5-49f9-9a71-b070b16b94fd (/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02) {'endpoint': 'v2.upload_chunk', 'request_id': 'urn:request:f16b264c-92d5-49f9-9a71-b070b16b94fd', 'remote_addr': '10.129.2.10', 'http_method': 'PATCH', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02', 'path': '/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:27,913 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:27,913 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:27,913 [292] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:27 +0000] "PATCH /v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02 HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:27 +0000] "PATCH /v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02 HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" (1.113 20151502 1.112) builder stdout | 2025-11-04 09:33:27,915 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 2, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:27.909377\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} gunicorn-registry stdout | 2025-11-04 09:33:27,925 [292] [DEBUG] [app] Starting request: urn:request:f41dc8c9-b075-43e4-8bdb-d0d96f1dff32 (/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:27,925 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'upload_uuid': '6256983b-3cf4-4041-9d44-3b3f77944f02', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:27,925 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:27,925 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:27,926 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:27,926 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:27,926 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:27,926 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:27,926 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:27,927 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,930 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,934 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,937 [292] [DEBUG] [peewee] ('SELECT Sum("t1"."byte_count") AS "size_bytes" FROM "blobupload" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,940 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,944 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,947 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,950 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:27,953 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['6256983b-3cf4-4041-9d44-3b3f77944f02', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:27,956 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:27,957 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:27,957 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:27,957 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [boto3.resources.action] Calling s3:create_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'} gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,958 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CreateMultipartUpload) with params: {'url_path': '/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2?uploads', 'query_string': {}, 'method': 'POST', 'headers': {'Content-Type': 'application/octet-stream', 'x-amz-server-side-encryption': 'AES256', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2?uploads', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2 gunicorn-registry stdout | uploads= gunicorn-registry stdout | content-type:application/octet-stream gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093327Z gunicorn-registry stdout | x-amz-server-side-encryption:AES256 gunicorn-registry stdout | content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093327Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 2d46fa3e9b4f3e0ef95032e6cfb57f590ff22e335b29327845ff2b62c08c7f78 gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 262109a7aec1e5c108854d1af3bf8921a0a22d4000afa5dd4ab3498176e9fc08 gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Type': b'application/octet-stream', 'x-amz-server-side-encryption': b'AES256', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093327Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption, Signature=262109a7aec1e5c108854d1af3bf8921a0a22d4000afa5dd4ab3498176e9fc08', 'amz-sdk-invocation-id': b'20d9d290-3660-40a3-baa4-5e8ea223f9f4', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:27,959 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:27,987 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2?uploads HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'V+aUXDlnQoM0ud7Qt1Ld6kzwkSK0lU2zoDRuc+YPSS72CDnB84a29mYB9MSrDpqoBFLzfPjlsVw=', 'x-amz-request-id': 'EFRJ6G9ERA15X6N1', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'x-amz-server-side-encryption': 'AES256', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\nwhuawsquaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd28T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6' gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'EFRJ6G9ERA15X6N1', 'HostId': 'V+aUXDlnQoM0ud7Qt1Ld6kzwkSK0lU2zoDRuc+YPSS72CDnB84a29mYB9MSrDpqoBFLzfPjlsVw=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'V+aUXDlnQoM0ud7Qt1Ld6kzwkSK0lU2zoDRuc+YPSS72CDnB84a29mYB9MSrDpqoBFLzfPjlsVw=', 'x-amz-request-id': 'EFRJ6G9ERA15X6N1', 'date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'x-amz-server-side-encryption': 'AES256', 'transfer-encoding': 'chunked', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'Bucket': 'whuaws', 'Key': 'quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'UploadId': '8T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6'} gunicorn-registry stdout | 2025-11-04 09:33:27,988 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUpload gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [boto3.resources.action] Calling s3:abort_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'UploadId': '8T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6'} gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-call.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-call.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.hooks] Event before-call.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,989 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=AbortMultipartUpload) with params: {'url_path': '/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'query_string': {'uploadId': '8T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6'}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2?uploadId=8T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2', 'UploadId': '8T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.hooks] Event request-created.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | DELETE gunicorn-registry stdout | /whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2 gunicorn-registry stdout | uploadId=8T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093327Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093327Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 50313b0e82d954e893b02ac5d5ea27f0254fc0a3bff3c971b9487b60ebbaa7e2 gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 97796e941cddfc38ac80ab4f0c50282213a5924ce60e34e5a74452112d9e1c8a gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.hooks] Event request-created.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.hooks] Event before-send.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093327Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=97796e941cddfc38ac80ab4f0c50282213a5924ce60e34e5a74452112d9e1c8a', 'amz-sdk-invocation-id': b'66985994-7aaa-4412-8c1e-990f082d8a79', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:27,990 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,029 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "DELETE /whuaws/quaydata/uploads/414ce6fd-6e0e-40d6-a4ad-349367c7bbd2?uploadId=8T0oAiBf0riKlveHg0uzzsf2NDsDSitnO.2ljUsl8Rbl64u27accohqyfaXgz3ajlTO308SvxXIIYbgZAFBYPLeF7tjCBbUuqS8LsXkYOf_1AIWoGBGtk_7Jt.90OZc6 HTTP/1.1" 204 0 gunicorn-registry stdout | 2025-11-04 09:33:28,029 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'oK5Sp5HByTBEoFSbk+rLcRpsNNwXSjI7ZYz7e73EPbvgponM26l9U1zZbgMmYgRRC5ZKD92ulZ8=', 'x-amz-request-id': 'EFRJ81KHYYQEYQJY', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [botocore.hooks] Event after-call.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'EFRJ81KHYYQEYQJY', 'HostId': 'oK5Sp5HByTBEoFSbk+rLcRpsNNwXSjI7ZYz7e73EPbvgponM26l9U1zZbgMmYgRRC5ZKD92ulZ8=', 'HTTPStatusCode': 204, 'HTTPHeaders': {'x-amz-id-2': 'oK5Sp5HByTBEoFSbk+rLcRpsNNwXSjI7ZYz7e73EPbvgponM26l9U1zZbgMmYgRRC5ZKD92ulZ8=', 'x-amz-request-id': 'EFRJ81KHYYQEYQJY', 'date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'server': 'AmazonS3'}, 'RetryAttempts': 0}} gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [data.registry_model.blobuploader] Uploaded 0 bytes to blob 6256983b-3cf4-4041-9d44-3b3f77944f02 took 0.0733041763305664 seconds gunicorn-registry stdout | 2025-11-04 09:33:28,030 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['6256983b-3cf4-4041-9d44-3b3f77944f02', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,035 [292] [DEBUG] [peewee] ('UPDATE "blobupload" SET "repository_id" = %s, "uuid" = %s, "byte_count" = %s, "sha_state" = %s, "location_id" = %s, "storage_metadata" = %s, "chunk_count" = %s, "uncompressed_byte_count" = %s, "created" = %s, "piece_sha_state" = %s, "piece_hashes" = %s WHERE ("blobupload"."id" = %s)', [2, '6256983b-3cf4-4041-9d44-3b3f77944f02', 20144803, 'gASVlQAAAAAAAACMD3Jlc3VtYWJsZXNoYTI1NpSMBnNoYTI1NpSTlCmBlENwCP+/X9fTpHF1Q3D4q3B4AAB44j8AAAD//wMAfk7Q3wD87AEcN30AZ87DdJ0135v+UyfnVf+n/vqP3zBiymj4fyMAAAAAAAAAABSbCQAAAACTVTzW/AOOZemeBs+hla001Ujx2TncHCfW27Z0gtNbAJRiLg==', 9, '{"chunks": [["uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2", 0, 20144803]]}', 2, 32308224, datetime.datetime(2025, 11, 4, 9, 33, 26, 775929), None, None, 4]) gunicorn-registry stdout | 2025-11-04 09:33:28,039 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:28,039 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:28,039 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0'} gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,040 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | HEAD gunicorn-registry stdout | /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 00240c1cedd5a14831bbf0a026cc0acc99a503d69f8338c6e10957e21bbb8710 gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 147f7216e51b54fc168a776a2116069838c61c90d5655fade8b8195b1b0d226e gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,041 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,042 [292] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,042 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=147f7216e51b54fc168a776a2116069838c61c90d5655fade8b8195b1b0d226e', 'amz-sdk-invocation-id': b'107fe277-b5d9-45e2-af86-dc7c621d487e', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,042 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "HEAD /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 HTTP/1.1" 404 0 gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-request-id': '12W87ZQ3WSF4E2XC', 'x-amz-id-2': '0iVb0zVpVMp2dfoRlPNbAFuuzQxdSNRQaSKQb4/xHLtWUBsJcjQ7PjxtUAa7UGtWaS8l+qZneLs=', 'Content-Type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Date': 'Tue, 04 Nov 2025 09:33:27 GMT', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,052 [292] [DEBUG] [storage.cloud] Performing server side assembly of multi-part upload for: sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [s3transfer.tasks] CopySubmissionTask(transfer_id=0, {'transfer_future': }) about to wait for the following futures [] gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [s3transfer.tasks] CopySubmissionTask(transfer_id=0, {'transfer_future': }) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [s3transfer.tasks] Executing task CopySubmissionTask(transfer_id=0, {'transfer_future': }) with kwargs {'client': , 'config': , 'osutil': , 'request_executor': , 'transfer_future': } gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,053 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | HEAD gunicorn-registry stdout | /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 45a207056868a64379052aa6022b5db291fe9bde81046d6d5f4ed6f7a1f3c07a gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 950e263a76eae418b3c4b4576d031c2a3d9654437260a0fef6984cb3d567e979 gunicorn-registry stdout | 2025-11-04 09:33:28,054 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,055 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,055 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,055 [292] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,055 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=950e263a76eae418b3c4b4576d031c2a3d9654437260a0fef6984cb3d567e979', 'amz-sdk-invocation-id': b'e13a3ad5-795f-4d08-b689-d2ee4f223c55', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,055 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,055 [292] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: s3.us-east-1.amazonaws.com gunicorn-registry stdout | 2025-11-04 09:33:28,106 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "HEAD /whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:28,106 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'h9TwlfIackwTEyhb107t8PHQQf7Me0a2ejcqchEFYWEAiHa0m8rNk8ewp8oYaOGSvi6Oih8HfT4=', 'x-amz-request-id': '12W6WWM54CGTP1ZB', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'Last-Modified': 'Tue, 04 Nov 2025 09:33:27 GMT', 'ETag': '"257795cb3db8363d60a9d07d48724faa-4"', 'x-amz-server-side-encryption': 'AES256', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/octet-stream', 'Content-Length': '20144803', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,107 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:28,107 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,107 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,107 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,107 [292] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,107 [292] [DEBUG] [s3transfer.futures] Submitting task CreateMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) to executor for transfer request: 0. gunicorn-registry stdout | 2025-11-04 09:33:28,107 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [s3transfer.tasks] CreateMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) about to wait for the following futures [] gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [s3transfer.tasks] CreateMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [s3transfer.tasks] Executing task CreateMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) with kwargs {'client': , 'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}} gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,108 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CreateMultipartUpload) with params: {'url_path': '/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploads', 'query_string': {}, 'method': 'POST', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploads', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 gunicorn-registry stdout | uploads= gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | c932401a8e0186db25d0418a9af99ef180d95e5e69bfd3c94c0a7c48b5a0addc gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 8114041acd398046c826d6bdbd7b38a0d788934960256f9ad48dddd0a9227f04 gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,109 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8114041acd398046c826d6bdbd7b38a0d788934960256f9ad48dddd0a9227f04', 'amz-sdk-invocation-id': b'fbf134a8-22b1-42d7-920b-ba0c30f415d8', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,110 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,110 [292] [DEBUG] [s3transfer.futures] Submitting task CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 1, 'extra_args': {'CopySourceRange': 'bytes=0-8388607'}}) to executor for transfer request: 0. gunicorn-registry stdout | 2025-11-04 09:33:28,110 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:28,110 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 1, 'extra_args': {'CopySourceRange': 'bytes=0-8388607'}}) about to wait for the following futures [] gunicorn-registry stdout | 2025-11-04 09:33:28,110 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 1, 'extra_args': {'CopySourceRange': 'bytes=0-8388607'}}) about to wait for gunicorn-registry stdout | 2025-11-04 09:33:28,110 [292] [DEBUG] [s3transfer.futures] Submitting task CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 2, 'extra_args': {'CopySourceRange': 'bytes=8388608-16777215'}}) to executor for transfer request: 0. gunicorn-registry stdout | 2025-11-04 09:33:28,110 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 2, 'extra_args': {'CopySourceRange': 'bytes=8388608-16777215'}}) about to wait for the following futures [] gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 2, 'extra_args': {'CopySourceRange': 'bytes=8388608-16777215'}}) about to wait for gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.futures] Submitting task CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 3, 'extra_args': {'CopySourceRange': 'bytes=16777216-20144802'}}) to executor for transfer request: 0. gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 3, 'extra_args': {'CopySourceRange': 'bytes=16777216-20144802'}}) about to wait for the following futures [] gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 3, 'extra_args': {'CopySourceRange': 'bytes=16777216-20144802'}}) about to wait for gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.futures] Submitting task CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) to executor for transfer request: 0. gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.tasks] CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) about to wait for the following futures [, , , ] gunicorn-registry stdout | 2025-11-04 09:33:28,111 [292] [DEBUG] [s3transfer.tasks] CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) about to wait for gunicorn-registry stdout | 2025-11-04 09:33:28,112 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None gunicorn-registry stdout | 2025-11-04 09:33:28,139 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploads HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'H3DbuKXSyCp22FSTJdVLQBEOzvZz6epBgTGxlsCVmsGyEuizpT51zdEOlIZiCKP2pFWegN/EETc=', 'x-amz-request-id': '12W50N23V0D3D7EA', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\nwhuawsquaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc01MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP' gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None gunicorn-registry stdout | 2025-11-04 09:33:28,140 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 1, 'extra_args': {'CopySourceRange': 'bytes=0-8388607'}}) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [s3transfer.tasks] Executing task CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 1, 'extra_args': {'CopySourceRange': 'bytes=0-8388607'}}) with kwargs {'client': , 'copy_source': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2'}, 'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 1, 'extra_args': {'CopySourceRange': 'bytes=0-8388607'}, 'callbacks': [], 'size': 8388608, 'checksum_algorithm': None, 'upload_id': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP'} gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,141 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPartCopy) with params: {'url_path': '/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'query_string': {'uploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'partNumber': 1}, 'method': 'PUT', 'headers': {'x-amz-copy-source': 'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'x-amz-copy-source-range': 'bytes=0-8388607', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP&partNumber=1', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'CopySource': 'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'Bucket': 'whuaws', 'Key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'UploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'PartNumber': 1, 'CopySourceRange': 'bytes=0-8388607'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 gunicorn-registry stdout | partNumber=1&uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-copy-source:whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | x-amz-copy-source-range:bytes=0-8388607 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-copy-source;x-amz-copy-source-range;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 99a5b3caf2defb46a30b8172d6c4ff733b468e96319008f216c4de5a1ce97f7a gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 8b5036e275da0dd1ca97ffd0d9df2bf2a6090676ca394b9feacd39cc603e38c0 gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,142 [292] [ERROR] [storage.cloud] [OTEL] request {'x-amz-copy-source': b'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'x-amz-copy-source-range': b'bytes=0-8388607', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-copy-source-range;x-amz-date, Signature=8b5036e275da0dd1ca97ffd0d9df2bf2a6090676ca394b9feacd39cc603e38c0', 'amz-sdk-invocation-id': b'b3335188-6086-4b8c-b943-77d42d9cb7ed', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 2, 'extra_args': {'CopySourceRange': 'bytes=8388608-16777215'}}) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [s3transfer.tasks] Executing task CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 2, 'extra_args': {'CopySourceRange': 'bytes=8388608-16777215'}}) with kwargs {'client': , 'copy_source': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2'}, 'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 2, 'extra_args': {'CopySourceRange': 'bytes=8388608-16777215'}, 'callbacks': [], 'size': 8388608, 'checksum_algorithm': None, 'upload_id': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP'} gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,143 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPartCopy) with params: {'url_path': '/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'query_string': {'uploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'partNumber': 2}, 'method': 'PUT', 'headers': {'x-amz-copy-source': 'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'x-amz-copy-source-range': 'bytes=8388608-16777215', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP&partNumber=2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'CopySource': 'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'Bucket': 'whuaws', 'Key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'UploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'PartNumber': 2, 'CopySourceRange': 'bytes=8388608-16777215'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 gunicorn-registry stdout | partNumber=2&uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-copy-source:whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | x-amz-copy-source-range:bytes=8388608-16777215 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-copy-source;x-amz-copy-source-range;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | aee102cbd5a1585e48cb7e1593d7758840ba1cace1f0c4bbafd65e2e692f0fe0 gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | b8bd4f767bde3c130b45c8d3dcfabca541f0232521c5dd71ec42d3c9d5f4a850 gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,144 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [ERROR] [storage.cloud] [OTEL] request {'x-amz-copy-source': b'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'x-amz-copy-source-range': b'bytes=8388608-16777215', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-copy-source-range;x-amz-date, Signature=b8bd4f767bde3c130b45c8d3dcfabca541f0232521c5dd71ec42d3c9d5f4a850', 'amz-sdk-invocation-id': b'4588979a-9bab-4588-8fcb-248955d3b573', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [s3transfer.tasks] CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 3, 'extra_args': {'CopySourceRange': 'bytes=16777216-20144802'}}) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [s3transfer.tasks] Executing task CopyPartTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 3, 'extra_args': {'CopySourceRange': 'bytes=16777216-20144802'}}) with kwargs {'client': , 'copy_source': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2'}, 'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'part_number': 3, 'extra_args': {'CopySourceRange': 'bytes=16777216-20144802'}, 'callbacks': [], 'size': 3367587, 'checksum_algorithm': None, 'upload_id': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP'} gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,145 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPartCopy) with params: {'url_path': '/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'query_string': {'uploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'partNumber': 3}, 'method': 'PUT', 'headers': {'x-amz-copy-source': 'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'x-amz-copy-source-range': 'bytes=16777216-20144802', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP&partNumber=3', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'CopySource': 'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'Bucket': 'whuaws', 'Key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'UploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'PartNumber': 3, 'CopySourceRange': 'bytes=16777216-20144802'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,146 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 gunicorn-registry stdout | partNumber=3&uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-copy-source:whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2 gunicorn-registry stdout | x-amz-copy-source-range:bytes=16777216-20144802 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-copy-source;x-amz-copy-source-range;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | b04b87c638e6902cd084048937c9b2c6638960f6cad488173819c5ab9413f875 gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | c42f7a0b8ed956085c79049fdc50c17de05c1a9cbbd0e2f888857a13a67d80a2 gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [ERROR] [storage.cloud] [OTEL] request {'x-amz-copy-source': b'whuaws/quaydata/uploads/1cdaf5fd-787f-473a-bddb-ee9ba1725bc2', 'x-amz-copy-source-range': b'bytes=16777216-20144802', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-copy-source-range;x-amz-date, Signature=c42f7a0b8ed956085c79049fdc50c17de05c1a9cbbd0e2f888857a13a67d80a2', 'amz-sdk-invocation-id': b'5a48d208-5e6b-4ef2-b923-1106508a0756', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,147 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,148 [292] [DEBUG] [s3transfer.tasks] CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) about to wait for gunicorn-registry stdout | 2025-11-04 09:33:28,148 [292] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: s3.us-east-1.amazonaws.com gunicorn-registry stdout | 2025-11-04 09:33:28,148 [292] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: s3.us-east-1.amazonaws.com logrotateworker stdout | 2025-11-04 09:33:28,251 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:28,337 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP&partNumber=1 HTTP/1.1" 200 220 gunicorn-registry stdout | 2025-11-04 09:33:28,337 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'VwzthePWiBd1ESAXqe27RxxXJ50PHFaXnQcQx2kAava2GRAMNpT4cVN6dnMh9jaTpVDBu9RfWKo=', 'x-amz-request-id': '12W978G53NCD5XEZ', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'Content-Type': 'application/xml', 'Content-Length': '220', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,337 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\n2025-11-04T09:33:29.000Z"d865a6c7b9cdac9372e5359754873c98"' gunicorn-registry stdout | 2025-11-04 09:33:28,338 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,338 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,338 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,338 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,338 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,338 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None gunicorn-registry stdout | 2025-11-04 09:33:28,338 [292] [DEBUG] [s3transfer.tasks] CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) about to wait for gunicorn-registry stdout | 2025-11-04 09:33:28,357 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP&partNumber=3 HTTP/1.1" 200 220 gunicorn-registry stdout | 2025-11-04 09:33:28,357 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': '1UMwpS5I2LTsfb/4Xu6T3lP8Tl0KExGW6Ak7I4qWo+qppfeHz7xaufQ2+BWK7yN5lPKcxOon1ac=', 'x-amz-request-id': '12WCM4R1Z4RBFFF4', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'Content-Type': 'application/xml', 'Content-Length': '220', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,357 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\n2025-11-04T09:33:29.000Z"3adc2358ce679209d0d58e9d3ab16d75"' gunicorn-registry stdout | 2025-11-04 09:33:28,357 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,357 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,358 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,358 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,358 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,358 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None gunicorn-registry stdout | 2025-11-04 09:33:28,428 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP&partNumber=2 HTTP/1.1" 200 220 gunicorn-registry stdout | 2025-11-04 09:33:28,428 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': '3PHBBwwRJiXvrX2KEg1MRMLELUcKOgB+x8IMDTwkZwEz6jHy+O78LABPUhOnBjK7qd1ZnxD3iTY=', 'x-amz-request-id': '12WBZZ9SXSR3D0VE', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'Content-Type': 'application/xml', 'Content-Length': '220', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,428 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\n2025-11-04T09:33:29.000Z"ecc92762094bd26b14e33756190d880c"' gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPartCopy: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [s3transfer.tasks] CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) about to wait for gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [s3transfer.tasks] CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [s3transfer.tasks] Executing task CompleteMultipartUploadTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}}) with kwargs {'client': , 'bucket': 'whuaws', 'key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'extra_args': {}, 'upload_id': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'parts': [{'ETag': '"d865a6c7b9cdac9372e5359754873c98"', 'PartNumber': 1}, {'ETag': '"ecc92762094bd26b14e33756190d880c"', 'PartNumber': 2}, {'ETag': '"3adc2358ce679209d0d58e9d3ab16d75"', 'PartNumber': 3}]} gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,429 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CompleteMultipartUpload) with params: {'url_path': '/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'query_string': {'uploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP'}, 'method': 'POST', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'"d865a6c7b9cdac9372e5359754873c98"1"ecc92762094bd26b14e33756190d880c"2"3adc2358ce679209d0d58e9d3ab16d75"3', 'auth_path': '/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'UploadId': '1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP', 'MultipartUpload': {'Parts': [{'ETag': '"d865a6c7b9cdac9372e5359754873c98"', 'PartNumber': 1}, {'ETag': '"ecc92762094bd26b14e33756190d880c"', 'PartNumber': 2}, {'ETag': '"3adc2358ce679209d0d58e9d3ab16d75"', 'PartNumber': 3}]}}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,430 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 gunicorn-registry stdout | uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:6fd8060531f8f9a1db63afcbd09c9d405c6ecebca0aabfcc8f97acbe822c3a32 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | 6fd8060531f8f9a1db63afcbd09c9d405c6ecebca0aabfcc8f97acbe822c3a32 gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | e5c82a6ac457cd853075123db598a462ff54dfc406375e03ffcaab41e7c1ca79 gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | ba8195bbc7308a45466cf9079a5308e13a67c5f54d75653dbeddfa7902a3206a gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'6fd8060531f8f9a1db63afcbd09c9d405c6ecebca0aabfcc8f97acbe822c3a32', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ba8195bbc7308a45466cf9079a5308e13a67c5f54d75653dbeddfa7902a3206a', 'amz-sdk-invocation-id': b'65442757-ca9b-4f54-a0eb-489458e1b27d', 'amz-sdk-request': b'attempt=1', 'Content-Length': '357', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,431 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,487 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0?uploadId=1MUOHdgRjTtLCnNpWv_SnN8W2nwaUcYzAi1cdV2GNGaN7IKdt2XCJOvmdkmK2wm74qM4HiXRcxE2D_m8X7rzKoKahhE4w2jfh5DXQR4X34fhw6mk3AF9bSCnD0t2oidP HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:28,617 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'CTZClMlKuHPfK94edQaYq/qzkvi4hU8toM7mjsBNUj2Rv9tRioug/lp3s+IRDV3/AGzV1jg+wnc=', 'x-amz-request-id': '12W333Q721K5R3HE', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'Content-Type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,617 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\n\nhttps://s3.us-east-1.amazonaws.com/whuaws/quaydata%2Fsha256%2F03%2F03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0whuawsquaydata/sha256/03/03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0"c26b4b02b307a8f14893ca918d7a44b1-3"N/avDMYf5jE=FULL_OBJECT' gunicorn-registry stdout | 2025-11-04 09:33:28,618 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,618 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,618 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,618 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,618 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,618 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None gunicorn-registry stdout | 2025-11-04 09:33:28,619 [292] [DEBUG] [data.registry_model.blobuploader] Completed chunked upload for blob 6256983b-3cf4-4041-9d44-3b3f77944f02 with digest sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 took 0.5664315223693848 seconds gunicorn-registry stdout | 2025-11-04 09:33:28,619 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['6256983b-3cf4-4041-9d44-3b3f77944f02', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,622 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."image_size", "t1"."uncompressed_size", "t1"."uploading", "t1"."cas_path", "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s OFFSET %s', ['sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,624 [292] [DEBUG] [peewee] ('INSERT INTO "imagestorage" ("uuid", "image_size", "uncompressed_size", "uploading", "cas_path", "content_checksum") VALUES (%s, %s, %s, %s, %s, %s) RETURNING "imagestorage"."id"', ['a5f4bc82-527d-404a-9094-8b5e424ebbbf', 20144803, 32308224, True, True, 'sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0']) gunicorn-registry stdout | 2025-11-04 09:33:28,625 [292] [DEBUG] [peewee] ('INSERT INTO "imagestorageplacement" ("storage_id", "location_id") VALUES (%s, %s) RETURNING "imagestorageplacement"."id"', [5, 9]) gunicorn-registry stdout | 2025-11-04 09:33:28,627 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,628 [292] [DEBUG] [peewee] ('INSERT INTO "uploadedblob" ("repository_id", "blob_id", "uploaded_at", "expires_at") VALUES (%s, %s, %s, %s) RETURNING "uploadedblob"."id"', [2, 5, datetime.datetime(2025, 11, 4, 9, 33, 28, 628397), datetime.datetime(2025, 11, 4, 10, 33, 28, 628384)]) gunicorn-registry stdout | 2025-11-04 09:33:28,629 [292] [DEBUG] [peewee] ('DELETE FROM "blobupload" WHERE ("blobupload"."id" = %s)', [4]) gunicorn-registry stdout | 2025-11-04 09:33:28,633 [292] [DEBUG] [app] Ending request: urn:request:f41dc8c9-b075-43e4-8bdb-d0d96f1dff32 (/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02) {'endpoint': 'v2.monolithic_upload_or_last_chunk', 'request_id': 'urn:request:f41dc8c9-b075-43e4-8bdb-d0d96f1dff32', 'remote_addr': '10.129.2.10', 'http_method': 'PUT', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02?digest=sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'path': '/v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02', 'parameters': {'digest': 'sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0'}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:28,634 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:28,634 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:28,634 [292] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:28 +0000] "PUT /v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02?digest=sha256%3A03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 HTTP/1.1" 201 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:28 +0000] "PUT /v2/build_test/upload_file_firefox/blobs/uploads/6256983b-3cf4-4041-9d44-3b3f77944f02?digest=sha256%3A03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0 HTTP/1.1" 201 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.710 1726 0.709) gunicorn-registry stdout | 2025-11-04 09:33:28,652 [292] [DEBUG] [app] Starting request: urn:request:fc04c690-2cac-4695-9d75-2d99c270e0b9 (/v2/build_test/upload_file_firefox/blobs/sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:28,652 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'digest': 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:28,652 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:28,652 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:28,653 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:28,653 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:28,653 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:28,653 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:28,653 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:28,654 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,658 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['build_test', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,661 [292] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['build_test', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:33:28,661 [292] [DEBUG] [data.cache.impl] Checking cache for key repo_blob__build_test_upload_file_firefox_sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8_2 gunicorn-registry stdout | 2025-11-04 09:33:28,661 [292] [DEBUG] [data.cache.impl] Found no result in cache for key repo_blob__build_test_upload_file_firefox_sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8_2; calling loader gunicorn-registry stdout | 2025-11-04 09:33:28,662 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,665 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,668 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,672 [292] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,675 [292] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,678 [292] [DEBUG] [data.cache.impl] Got loaded result for key repo_blob__build_test_upload_file_firefox_sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8_2: None gunicorn-registry stdout | 2025-11-04 09:33:28,678 [292] [DEBUG] [data.cache.impl] Not caching loaded result for key repo_blob__build_test_upload_file_firefox_sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8_2: None gunicorn-registry stdout | 2025-11-04 09:33:28,678 [292] [DEBUG] [endpoints.v2] sending response: b'{"errors":[{"code":"BLOB_UNKNOWN","detail":{},"message":"blob unknown to registry"}]}\n' gunicorn-registry stdout | 2025-11-04 09:33:28,678 [292] [DEBUG] [app] Ending request: urn:request:fc04c690-2cac-4695-9d75-2d99c270e0b9 (/v2/build_test/upload_file_firefox/blobs/sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8) {'endpoint': 'v2.check_blob_exists', 'request_id': 'urn:request:fc04c690-2cac-4695-9d75-2d99c270e0b9', 'remote_addr': '10.131.0.6', 'http_method': 'HEAD', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'path': '/v2/build_test/upload_file_firefox/blobs/sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:28,678 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:28,678 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:28,679 [292] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:28 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 HTTP/1.1" 404 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:28 +0000] "HEAD /v2/build_test/upload_file_firefox/blobs/sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 HTTP/1.1" 404 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.027 1591 0.028) gunicorn-registry stdout | 2025-11-04 09:33:28,687 [292] [DEBUG] [app] Starting request: urn:request:29f2808a-aa9a-47fd-a515-e611f3581ee3 (/v2/build_test/upload_file_firefox/blobs/uploads/) {'X-Forwarded-For': '10.129.2.10'} gunicorn-registry stdout | 2025-11-04 09:33:28,687 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:28,687 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:28,688 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:28,688 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:28,688 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:28,688 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:28,688 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:28,688 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:28,689 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,692 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,695 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,699 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,702 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,705 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,709 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:28,712 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,715 [292] [DEBUG] [peewee] ('INSERT INTO "blobupload" ("repository_id", "uuid", "byte_count", "sha_state", "location_id", "storage_metadata", "chunk_count", "created") VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING "blobupload"."id"', [2, '46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 0, 'gASVlQAAAAAAAACMD3Jlc3VtYWJsZXNoYTI1NpSMBnNoYTI1NpSTlCmBlENwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW5RiLg==', 9, '{"chunks": []}', 0, datetime.datetime(2025, 11, 4, 9, 33, 28, 715146)]) gunicorn-registry stdout | 2025-11-04 09:33:28,720 [292] [DEBUG] [app] Ending request: urn:request:29f2808a-aa9a-47fd-a515-e611f3581ee3 (/v2/build_test/upload_file_firefox/blobs/uploads/) {'endpoint': 'v2.start_blob_upload', 'request_id': 'urn:request:29f2808a-aa9a-47fd-a515-e611f3581ee3', 'remote_addr': '10.129.2.10', 'http_method': 'POST', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/uploads/', 'path': '/v2/build_test/upload_file_firefox/blobs/uploads/', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:28,720 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:28,720 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:28,720 [292] [INFO] [gunicorn.access] 10.129.2.10 - - [04/Nov/2025:09:33:28 +0000] "POST /v2/build_test/upload_file_firefox/blobs/uploads/ HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:28 +0000] "POST /v2/build_test/upload_file_firefox/blobs/uploads/ HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.033 1570 0.033) gunicorn-registry stdout | 2025-11-04 09:33:28,731 [292] [DEBUG] [app] Starting request: urn:request:8284868f-dbb3-421c-aa0e-58d44f16c998 (/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:28,731 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'upload_uuid': '46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:28,731 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:28,731 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:28,732 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:28,732 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:28,732 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:28,732 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:28,732 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:28,732 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,736 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,739 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,742 [292] [DEBUG] [peewee] ('SELECT Sum("t1"."byte_count") AS "size_bytes" FROM "blobupload" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,745 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,749 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,752 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,755 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:28,758 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:28,762 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:28,762 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:28,762 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:28,762 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [boto3.resources.action] Calling s3:create_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'} gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CreateMultipartUpload) with params: {'url_path': '/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9?uploads', 'query_string': {}, 'method': 'POST', 'headers': {'Content-Type': 'application/octet-stream', 'x-amz-server-side-encryption': 'AES256', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9?uploads', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,763 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9 gunicorn-registry stdout | uploads= gunicorn-registry stdout | content-type:application/octet-stream gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | x-amz-server-side-encryption:AES256 gunicorn-registry stdout | content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | d55b7b89cb19f9cefc3279ec110edc8ff4f97749201aca290773d8ffad50c87c gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 5855acee980ee9d79e86b4314209067fab7265f710f44af63014b3ac998da6e3 gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Type': b'application/octet-stream', 'x-amz-server-side-encryption': b'AES256', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption, Signature=5855acee980ee9d79e86b4314209067fab7265f710f44af63014b3ac998da6e3', 'amz-sdk-invocation-id': b'b62721d6-e168-4bba-bfa4-bc1d6b7aa9e3', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,764 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,801 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9?uploads HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': '0T46zr/8HSY3FVLigzQQDf4Y9SZm57NgUSG2gutVhyLBzEpSxBDdLyz+vtp35t9HRFwCddKx6SQ=', 'x-amz-request-id': '12WDHFG2HNHKNWNJ', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\nwhuawsquaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS' gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': '12WDHFG2HNHKNWNJ', 'HostId': '0T46zr/8HSY3FVLigzQQDf4Y9SZm57NgUSG2gutVhyLBzEpSxBDdLyz+vtp35t9HRFwCddKx6SQ=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': '0T46zr/8HSY3FVLigzQQDf4Y9SZm57NgUSG2gutVhyLBzEpSxBDdLyz+vtp35t9HRFwCddKx6SQ=', 'x-amz-request-id': '12WDHFG2HNHKNWNJ', 'date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'transfer-encoding': 'chunked', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'UploadId': 'W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS'} gunicorn-registry stdout | 2025-11-04 09:33:28,802 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUpload gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUploadPart gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [boto3.resources.model] Renaming MultipartUploadPart attribute part_number gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [boto3.resources.action] Calling s3:upload_part with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'UploadId': 'W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS', 'PartNumber': 1, 'Body': <_io.BytesIO object at 0x7f679ca16750>, 'ContentLength': 7830} gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,803 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.handlers] Adding expect 100 continue header to request. gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event before-call.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=UploadPart) with params: {'url_path': '/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'query_string': {'uploadId': 'W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS', 'partNumber': 1}, 'method': 'PUT', 'headers': {'Content-Length': '7830', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': 'OkZMrBA2FxI7HUY7UfvaJg==', 'Expect': '100-continue'}, 'body': <_io.BytesIO object at 0x7f679ca16750>, 'auth_path': '/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9?uploadId=W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS&partNumber=1', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': True, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'UploadId': 'W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS', 'PartNumber': 1, 'Body': <_io.BytesIO object at 0x7f679ca16750>, 'ContentLength': 7830}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9 gunicorn-registry stdout | partNumber=1&uploadId=W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS gunicorn-registry stdout | content-length:7830 gunicorn-registry stdout | content-md5:OkZMrBA2FxI7HUY7UfvaJg== gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:UNSIGNED-PAYLOAD gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | content-length;content-md5;host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | UNSIGNED-PAYLOAD gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 10fb1969307b77b0a9ba6959f0f744caf6ad3a09dfb83fe0828d4928d6e9ee08 gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 39a79702c2e8bde75b2c47ae58f5c221e7533780e3685af082ac6d17f40a82f2 gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event request-created.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.hooks] Event before-send.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Length': b'7830', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'Content-MD5': b'OkZMrBA2FxI7HUY7UfvaJg==', 'Expect': b'100-continue', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'UNSIGNED-PAYLOAD', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date, Signature=39a79702c2e8bde75b2c47ae58f5c221e7533780e3685af082ac6d17f40a82f2', 'amz-sdk-invocation-id': b'8cac8ebb-19b0-4139-a0d8-625bcc7eb369', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,804 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:28,805 [292] [DEBUG] [botocore.awsrequest] Waiting for 100 Continue response. gunicorn-registry stdout | 2025-11-04 09:33:28,841 [292] [DEBUG] [botocore.awsrequest] 100 Continue response seen, now sending request body. gunicorn-registry stdout | 2025-11-04 09:33:28,867 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9?uploadId=W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS&partNumber=1 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:28,867 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'VgslQ8j+zdDX3eaOySIEbM7jlhlhQslKf+yvQjobSjIRNSG8Svb3d/9jbslK8rVrPEUFJvCooM8=', 'x-amz-request-id': '12W43ET4KBSR59ZX', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'ETag': '"3a464cac103617123b1d463b51fbda26"', 'x-amz-server-side-encryption': 'AES256', 'Content-Length': '0', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:28,867 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:28,867 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event after-call.s3.UploadPart: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': '12W43ET4KBSR59ZX', 'HostId': 'VgslQ8j+zdDX3eaOySIEbM7jlhlhQslKf+yvQjobSjIRNSG8Svb3d/9jbslK8rVrPEUFJvCooM8=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'VgslQ8j+zdDX3eaOySIEbM7jlhlhQslKf+yvQjobSjIRNSG8Svb3d/9jbslK8rVrPEUFJvCooM8=', 'x-amz-request-id': '12W43ET4KBSR59ZX', 'date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'etag': '"3a464cac103617123b1d463b51fbda26"', 'x-amz-server-side-encryption': 'AES256', 'content-length': '0', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'ETag': '"3a464cac103617123b1d463b51fbda26"'} gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [boto3.resources.action] Calling s3:complete_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'UploadId': 'W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS', 'MultipartUpload': {'Parts': [{'ETag': '"3a464cac103617123b1d463b51fbda26"', 'PartNumber': 1}]}} gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:28,868 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CompleteMultipartUpload) with params: {'url_path': '/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'query_string': {'uploadId': 'W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS'}, 'method': 'POST', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'"3a464cac103617123b1d463b51fbda26"1', 'auth_path': '/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9?uploadId=W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'UploadId': 'W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS', 'MultipartUpload': {'Parts': [{'ETag': '"3a464cac103617123b1d463b51fbda26"', 'PartNumber': 1}]}}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9 gunicorn-registry stdout | uploadId=W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:b2e55f2b9fff939b9ded67c77898bc8174964e6ba220bfb6be91bd9553ad0778 gunicorn-registry stdout | x-amz-date:20251104T093328Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | b2e55f2b9fff939b9ded67c77898bc8174964e6ba220bfb6be91bd9553ad0778 gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093328Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 61d0b09d357739a9f3bc431f2d40488de005e53ea67d21ceb553e8ee925702e5 gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 1843651ec1074533d79e84864add2399c582dbddb05e57413cb580ca3db87300 gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093328Z', 'X-Amz-Content-SHA256': b'b2e55f2b9fff939b9ded67c77898bc8174964e6ba220bfb6be91bd9553ad0778', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1843651ec1074533d79e84864add2399c582dbddb05e57413cb580ca3db87300', 'amz-sdk-invocation-id': b'8e020114-53c4-49ad-a57f-2c1bee21cdb7', 'amz-sdk-request': b'attempt=1', 'Content-Length': '185', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:28,869 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem storagereplication stdout | 2025-11-04 09:33:28,903 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:28,986 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9?uploadId=W7f5QyjBqyPpebu79K46aMZaacW6UUWARh0CjqnqX1HXii0zvuCmoWyVAdyQOdmWR6FNVLAGICyAdZ3dCrG3_Km3hTc1J0LylFAhJUoAdSI3x9gbuCnh2G3_wKOBJQcS HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:29,048 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'fs5FWay5qdpUInOdKLpyS3sqWr8t1sotdXruHfKe/pjKcKThvRKCtwo3D/zQqizWjGt1mufsTyI=', 'x-amz-request-id': '12W8V1328CK22VM4', 'Date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'Content-Type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:29,048 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\n\nhttps://s3.us-east-1.amazonaws.com/whuaws/quaydata%2Fuploads%2F5abb22b0-b029-4692-8906-d0b6937135f9whuawsquaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9"3b03fefd38c318c7ac6b394ec4b9308d-1"T1Ls5Cdk0Ic=FULL_OBJECT' gunicorn-registry stdout | 2025-11-04 09:33:29,048 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,048 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,049 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:29,049 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,049 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CompleteMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,049 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': '12W8V1328CK22VM4', 'HostId': 'fs5FWay5qdpUInOdKLpyS3sqWr8t1sotdXruHfKe/pjKcKThvRKCtwo3D/zQqizWjGt1mufsTyI=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'fs5FWay5qdpUInOdKLpyS3sqWr8t1sotdXruHfKe/pjKcKThvRKCtwo3D/zQqizWjGt1mufsTyI=', 'x-amz-request-id': '12W8V1328CK22VM4', 'date': 'Tue, 04 Nov 2025 09:33:29 GMT', 'x-amz-server-side-encryption': 'AES256', 'content-type': 'application/xml', 'transfer-encoding': 'chunked', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'Location': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata%2Fuploads%2F5abb22b0-b029-4692-8906-d0b6937135f9', 'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'ETag': '"3b03fefd38c318c7ac6b394ec4b9308d-1"'} gunicorn-registry stdout | 2025-11-04 09:33:29,049 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:29,050 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:29,050 [292] [DEBUG] [data.registry_model.blobuploader] Uploaded 7830 bytes to blob 46f2369c-89d6-4358-96f1-fc0bd1f0ae4f took 0.28826355934143066 seconds gunicorn-registry stdout | 2025-11-04 09:33:29,050 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,054 [292] [DEBUG] [peewee] ('UPDATE "blobupload" SET "repository_id" = %s, "uuid" = %s, "byte_count" = %s, "sha_state" = %s, "location_id" = %s, "storage_metadata" = %s, "chunk_count" = %s, "uncompressed_byte_count" = %s, "created" = %s, "piece_sha_state" = %s, "piece_hashes" = %s WHERE ("blobupload"."id" = %s)', [2, '46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 7830, 'gASVlQAAAAAAAACMD3Jlc3VtYWJsZXNoYTI1NpSMBnNoYTI1NpSTlCmBlENwb20vdWJpNy91Ymk6bGF0ZXN0In1dfSIsImNvbW1lbnQiOiJGUk9NIHJlZ2lzdHJ5LmFjY2Vzcy5yZWRoYXQuYxYAAAAAAAAAAPQAAAAAAADM1dgpaIn8wR9K6lgA53ZE+/xQdLuQYjktR3nJtAQGrpRiLg==', 9, '{"chunks": [["uploads/5abb22b0-b029-4692-8906-d0b6937135f9", 0, 7830]]}', 1, None, datetime.datetime(2025, 11, 4, 9, 33, 28, 715146), None, None, 5]) gunicorn-registry stdout | 2025-11-04 09:33:29,060 [292] [DEBUG] [app] Ending request: urn:request:8284868f-dbb3-421c-aa0e-58d44f16c998 (/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f) {'endpoint': 'v2.upload_chunk', 'request_id': 'urn:request:8284868f-dbb3-421c-aa0e-58d44f16c998', 'remote_addr': '10.131.0.6', 'http_method': 'PATCH', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 'path': '/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:29,060 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:29,060 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:29,060 [292] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:29 +0000] "PATCH /v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:29 +0000] "PATCH /v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f HTTP/1.1" 202 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.332 9480 0.329) gunicorn-registry stdout | 2025-11-04 09:33:29,073 [292] [DEBUG] [app] Starting request: urn:request:c711b56a-c100-466b-b40d-84a33face524 (/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:29,073 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'upload_uuid': '46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:29,073 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:29,073 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:29,074 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:29,074 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:29,074 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:29,074 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:29,074 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:29,075 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,078 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,082 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,085 [292] [DEBUG] [peewee] ('SELECT Sum("t1"."byte_count") AS "size_bytes" FROM "blobupload" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,088 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,091 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,094 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,098 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:29,101 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,104 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:29,104 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:29,104 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [boto3.resources.action] Calling s3:create_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'} gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,105 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CreateMultipartUpload) with params: {'url_path': '/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab?uploads', 'query_string': {}, 'method': 'POST', 'headers': {'Content-Type': 'application/octet-stream', 'x-amz-server-side-encryption': 'AES256', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab?uploads', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | POST gunicorn-registry stdout | /whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab gunicorn-registry stdout | uploads= gunicorn-registry stdout | content-type:application/octet-stream gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093329Z gunicorn-registry stdout | x-amz-server-side-encryption:AES256 gunicorn-registry stdout | content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093329Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | e11aaed08ed19161ca2909da76da9c973fd9796254f3469007486e987cde90b6 gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | ea92dd3e59f404b8b835139aa80c86f2f8f41e7175e68280e03267a07854199b gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [ERROR] [storage.cloud] [OTEL] request {'Content-Type': b'application/octet-stream', 'x-amz-server-side-encryption': b'AES256', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093329Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption, Signature=ea92dd3e59f404b8b835139aa80c86f2f8f41e7175e68280e03267a07854199b', 'amz-sdk-invocation-id': b'12abc39f-b919-4dd9-a037-36e9e63e5f63', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:29,106 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:29,195 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "POST /whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab?uploads HTTP/1.1" 200 None gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': '6At8lJ+xP4PKDY/dCKmV5pwAj0GXDefCIsjILiEtduSp4VpS3ix9Ccb+3ypuoLBvfKoGmFwJ1dM=', 'x-amz-request-id': 'R4EX62AYGR471J28', 'Date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'x-amz-server-side-encryption': 'AES256', 'Transfer-Encoding': 'chunked', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\nwhuawsquaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44abgAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR' gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CreateMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'R4EX62AYGR471J28', 'HostId': '6At8lJ+xP4PKDY/dCKmV5pwAj0GXDefCIsjILiEtduSp4VpS3ix9Ccb+3ypuoLBvfKoGmFwJ1dM=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': '6At8lJ+xP4PKDY/dCKmV5pwAj0GXDefCIsjILiEtduSp4VpS3ix9Ccb+3ypuoLBvfKoGmFwJ1dM=', 'x-amz-request-id': 'R4EX62AYGR471J28', 'date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'x-amz-server-side-encryption': 'AES256', 'transfer-encoding': 'chunked', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'ServerSideEncryption': 'AES256', 'Bucket': 'whuaws', 'Key': 'quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'UploadId': 'gAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR'} gunicorn-registry stdout | 2025-11-04 09:33:29,196 [292] [DEBUG] [boto3.resources.factory] Loading s3:MultipartUpload gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [boto3.resources.action] Calling s3:abort_multipart_upload with {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'UploadId': 'gAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR'} gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-call.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-call.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-call.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=AbortMultipartUpload) with params: {'url_path': '/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'query_string': {'uploadId': 'gAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR'}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab?uploadId=gAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab', 'UploadId': 'gAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event request-created.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,197 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | DELETE gunicorn-registry stdout | /whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab gunicorn-registry stdout | uploadId=gAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093329Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093329Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 5e8adfad06d405b4da891090fcc84895a20e2a0bbc6f19ca67fec4db96457a18 gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | f6f76048f24b3516dd70abf281290ea90af29187d625992d8ea9c81a78a2351e gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.hooks] Event request-created.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.hooks] Event before-send.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093329Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=f6f76048f24b3516dd70abf281290ea90af29187d625992d8ea9c81a78a2351e', 'amz-sdk-invocation-id': b'690effd1-188d-4c56-af9d-280361e8532f', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:29,198 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:29,234 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "DELETE /whuaws/quaydata/uploads/e9e58797-45e0-4247-9ad8-386bc59f44ab?uploadId=gAuMgI7uhbDwtsVWdnQ8JRIbAQaqsATK8610kBjaUOn7DRq716lEQPHD0ENNSly9QBFeWVUOkEYLdyqxKEEBww9qGkcx_8mf.cPTQi80784.64N_8tEjWLEyGRcHArwR HTTP/1.1" 204 0 gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': '/7AWRYP/t41ZiP/Eetp86umzSf8YNAZlPLrA0IfIVEPoZg0D/SR8w97ukHNO4TspJwzJIZwg/Z8=', 'x-amz-request-id': 'R4EH42T5ZE3DKXSH', 'Date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.AbortMultipartUpload: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [botocore.hooks] Event after-call.s3.AbortMultipartUpload: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'R4EH42T5ZE3DKXSH', 'HostId': '/7AWRYP/t41ZiP/Eetp86umzSf8YNAZlPLrA0IfIVEPoZg0D/SR8w97ukHNO4TspJwzJIZwg/Z8=', 'HTTPStatusCode': 204, 'HTTPHeaders': {'x-amz-id-2': '/7AWRYP/t41ZiP/Eetp86umzSf8YNAZlPLrA0IfIVEPoZg0D/SR8w97ukHNO4TspJwzJIZwg/Z8=', 'x-amz-request-id': 'R4EH42T5ZE3DKXSH', 'date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'server': 'AmazonS3'}, 'RetryAttempts': 0}} gunicorn-registry stdout | 2025-11-04 09:33:29,235 [292] [DEBUG] [data.registry_model.blobuploader] Uploaded 0 bytes to blob 46f2369c-89d6-4358-96f1-fc0bd1f0ae4f took 0.13111400604248047 seconds gunicorn-registry stdout | 2025-11-04 09:33:29,236 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,239 [292] [DEBUG] [peewee] ('UPDATE "blobupload" SET "repository_id" = %s, "uuid" = %s, "byte_count" = %s, "sha_state" = %s, "location_id" = %s, "storage_metadata" = %s, "chunk_count" = %s, "uncompressed_byte_count" = %s, "created" = %s, "piece_sha_state" = %s, "piece_hashes" = %s WHERE ("blobupload"."id" = %s)', [2, '46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 7830, 'gASVlQAAAAAAAACMD3Jlc3VtYWJsZXNoYTI1NpSMBnNoYTI1NpSTlCmBlENwb20vdWJpNy91Ymk6bGF0ZXN0In1dfSIsImNvbW1lbnQiOiJGUk9NIHJlZ2lzdHJ5LmFjY2Vzcy5yZWRoYXQuYxYAAAAAAAAAAPQAAAAAAADM1dgpaIn8wR9K6lgA53ZE+/xQdLuQYjktR3nJtAQGrpRiLg==', 9, '{"chunks": [["uploads/5abb22b0-b029-4692-8906-d0b6937135f9", 0, 7830]]}', 2, None, datetime.datetime(2025, 11, 4, 9, 33, 28, 715146), None, None, 5]) gunicorn-registry stdout | 2025-11-04 09:33:29,244 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:29,244 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:29,244 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8'} gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:29,245 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | HEAD gunicorn-registry stdout | /whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093329Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093329Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | d632ffe5b969f650c732851dde5aacba07c6bb9856d25be9a839f1597749e21a gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 432c55c2db62353fa9987b1277f98a3efcb98ff37a1ba8ffb0a4542bfb3dc5d2 gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093329Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=432c55c2db62353fa9987b1277f98a3efcb98ff37a1ba8ffb0a4542bfb3dc5d2', 'amz-sdk-invocation-id': b'30ecd9c2-3a76-4a09-98d0-5afd31663be4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:29,246 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "HEAD /whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 HTTP/1.1" 404 0 gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-request-id': 'R4EZHP6KJF7P2YR2', 'x-amz-id-2': '+e+BEJ2Bk7KqTUJCYkaZtk/0MZXdmQzDVCSfxrV13tCJlKsDYtRScDg4r+DVcPJxciRGlAsOIBM=', 'Content-Type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Date': 'Tue, 04 Nov 2025 09:33:28 GMT', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,259 [292] [DEBUG] [storage.cloud] Performing server side assembly of multi-part upload for: sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [s3transfer.tasks] CopySubmissionTask(transfer_id=0, {'transfer_future': }) about to wait for the following futures [] gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [s3transfer.tasks] CopySubmissionTask(transfer_id=0, {'transfer_future': }) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [s3transfer.tasks] Executing task CopySubmissionTask(transfer_id=0, {'transfer_future': }) with kwargs {'client': , 'config': , 'osutil': , 'request_executor': , 'transfer_future': } gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,260 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | HEAD gunicorn-registry stdout | /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093329Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093329Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | b9e3e191ef87efbbb9a2b0383da8ff3ca7c17fa4f9011b4295e02cc2d7c5de4b gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | a28e82674d255cdc11de2da5c9fe0ded07f7300aae7f6789e5e76d0c6a870690 gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,261 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093329Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a28e82674d255cdc11de2da5c9fe0ded07f7300aae7f6789e5e76d0c6a870690', 'amz-sdk-invocation-id': b'010a192f-1e55-43d6-9db2-7e72d785597c', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:29,262 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:29,275 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "HEAD /whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:33:29,275 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'dxJVIDntcvrrTLxz5Uf8bzOBtD1ooGqQeLpPGdwdGen9n0LhlxAT3nefm1fWhHt8wrxM/LxIm7w=', 'x-amz-request-id': 'R4EWGN83YJGRV37F', 'Date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'Last-Modified': 'Tue, 04 Nov 2025 09:33:29 GMT', 'ETag': '"3b03fefd38c318c7ac6b394ec4b9308d-1"', 'x-amz-server-side-encryption': 'AES256', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/octet-stream', 'Content-Length': '7830', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:29,275 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:33:29,275 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [s3transfer.futures] Submitting task CopyObjectTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'extra_args': {}}) to executor for transfer request: 0. gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [s3transfer.utils] Acquiring 0 gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [s3transfer.tasks] CopyObjectTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'extra_args': {}}) about to wait for the following futures [] gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [s3transfer.tasks] CopyObjectTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'extra_args': {}}) done waiting for dependent futures gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [s3transfer.tasks] Executing task CopyObjectTask(transfer_id=0, {'bucket': 'whuaws', 'key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'extra_args': {}}) with kwargs {'client': , 'copy_source': {'Bucket': 'whuaws', 'Key': 'quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9'}, 'bucket': 'whuaws', 'key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'extra_args': {}, 'callbacks': [], 'size': 7830} gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,276 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event before-call.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=CopyObject) with params: {'url_path': '/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'query_string': {}, 'method': 'PUT', 'headers': {'x-amz-copy-source': 'whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'CopySource': 'whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'Bucket': 'whuaws', 'Key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:29,277 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CopyObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | PUT gunicorn-registry stdout | /whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-copy-source:whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9 gunicorn-registry stdout | x-amz-date:20251104T093329Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-copy-source;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093329Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 259f01bc2c84ea3f478853acbb538d39832f81234717a1c07b7e4423e66f5253 gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | f954012c0eea4c1c242ebec635beb41ec202af099b2eeff92dd7cefe2825212b gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.hooks] Event request-created.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.hooks] Event before-send.s3.CopyObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [ERROR] [storage.cloud] [OTEL] request {'x-amz-copy-source': b'whuaws/quaydata/uploads/5abb22b0-b029-4692-8906-d0b6937135f9', 'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T093329Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date, Signature=f954012c0eea4c1c242ebec635beb41ec202af099b2eeff92dd7cefe2825212b', 'amz-sdk-invocation-id': b'e562b93f-ff50-44e5-b3e5-dfea31b80e51', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:29,278 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:29,279 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None teamsyncworker stdout | 2025-11-04 09:33:29,286 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:29,316 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "PUT /whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 HTTP/1.1" 200 275 gunicorn-registry stdout | 2025-11-04 09:33:29,316 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': 'eera5uu7/M5BJWYAUZZ0qQC+xPUsD7fj5PRcJMp8iiEHzwSFKA1m0SPWEdqgd30q/tjGVh6yHmU=', 'x-amz-request-id': 'R4ESF6472C653EYA', 'Date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'x-amz-server-side-encryption': 'AES256', 'Content-Type': 'application/xml', 'Content-Length': '275', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:29,316 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'\n2025-11-04T09:33:30.000Z"3a464cac103617123b1d463b51fbda26"T1Ls5Cdk0Ic=' gunicorn-registry stdout | 2025-11-04 09:33:29,317 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,317 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CopyObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,317 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:29,317 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.CopyObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,317 [292] [DEBUG] [botocore.hooks] Event after-call.s3.CopyObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,317 [292] [DEBUG] [s3transfer.utils] Releasing acquire 0/None gunicorn-registry stdout | 2025-11-04 09:33:29,318 [292] [DEBUG] [data.registry_model.blobuploader] Completed chunked upload for blob 46f2369c-89d6-4358-96f1-fc0bd1f0ae4f with digest sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 took 0.058193206787109375 seconds gunicorn-registry stdout | 2025-11-04 09:33:29,318 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."uuid", "t1"."byte_count", "t1"."sha_state", "t1"."location_id", "t1"."storage_metadata", "t1"."chunk_count", "t1"."uncompressed_byte_count", "t1"."created", "t1"."piece_sha_state", "t1"."piece_hashes", "t2"."id", "t2"."name" FROM "blobupload" AS "t1" INNER JOIN "imagestoragelocation" AS "t2" ON ("t1"."location_id" = "t2"."id") WHERE ("t1"."uuid" = %s) LIMIT %s OFFSET %s', ['46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,321 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."image_size", "t1"."uncompressed_size", "t1"."uploading", "t1"."cas_path", "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s OFFSET %s', ['sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,322 [292] [DEBUG] [peewee] ('INSERT INTO "imagestorage" ("uuid", "image_size", "uncompressed_size", "uploading", "cas_path", "content_checksum") VALUES (%s, %s, %s, %s, %s, %s) RETURNING "imagestorage"."id"', ['6dd31602-c2a2-4ba8-9b54-5e1279d3f1f7', 7830, None, True, True, 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8']) gunicorn-registry stdout | 2025-11-04 09:33:29,323 [292] [DEBUG] [peewee] ('INSERT INTO "imagestorageplacement" ("storage_id", "location_id") VALUES (%s, %s) RETURNING "imagestorageplacement"."id"', [6, 9]) gunicorn-registry stdout | 2025-11-04 09:33:29,325 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,326 [292] [DEBUG] [peewee] ('INSERT INTO "uploadedblob" ("repository_id", "blob_id", "uploaded_at", "expires_at") VALUES (%s, %s, %s, %s) RETURNING "uploadedblob"."id"', [2, 6, datetime.datetime(2025, 11, 4, 9, 33, 29, 326450), datetime.datetime(2025, 11, 4, 10, 33, 29, 326438)]) gunicorn-registry stdout | 2025-11-04 09:33:29,327 [292] [DEBUG] [peewee] ('DELETE FROM "blobupload" WHERE ("blobupload"."id" = %s)', [5]) gunicorn-registry stdout | 2025-11-04 09:33:29,336 [292] [DEBUG] [app] Ending request: urn:request:c711b56a-c100-466b-b40d-84a33face524 (/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f) {'endpoint': 'v2.monolithic_upload_or_last_chunk', 'request_id': 'urn:request:c711b56a-c100-466b-b40d-84a33face524', 'remote_addr': '10.131.0.6', 'http_method': 'PUT', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f?digest=sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'path': '/v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f', 'parameters': {'digest': 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8'}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:29,337 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:29,337 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:29,337 [292] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:29 +0000] "PUT /v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f?digest=sha256%3A11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 HTTP/1.1" 201 0 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:29 +0000] "PUT /v2/build_test/upload_file_firefox/blobs/uploads/46f2369c-89d6-4358-96f1-fc0bd1f0ae4f?digest=sha256%3A11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 HTTP/1.1" 201 0 "-" "containers/5.29.5 (github.com/containers/image)" (0.265 1726 0.264) gunicorn-registry stdout | 2025-11-04 09:33:29,349 [292] [DEBUG] [app] Starting request: urn:request:771cd083-c5aa-4589-9717-f1853377ed9a (/v2/build_test/upload_file_firefox/manifests/latest) {'X-Forwarded-For': '10.131.0.6'} gunicorn-registry stdout | 2025-11-04 09:33:29,349 [292] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'manifest_ref': 'latest', 'namespace_name': 'build_test', 'repo_name': 'upload_file_firefox'} gunicorn-registry stdout | 2025-11-04 09:33:29,350 [292] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:29,350 [292] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6Im9TRGR4eHczbXQwZWozRWxHZzlMQmNMMmtveFdCdENuQXl2ZWxnNzZtWTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLmFwcHMud2h1NDE5YXowNi5xZS5henVyZS5kZXZjbHVzdGVyLm9wZW5zaGlmdC5jb20iLCJuYmYiOjE3NjIyNDg4MDYsImlhdCI6MTc2MjI0ODgwNiwiZXhwIjoxNzYyMjUyNDA2LCJzdWIiOm51bGwsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoiYnVpbGRfdGVzdC91cGxvYWRfZmlsZV9maXJlZm94IiwiYWN0aW9ucyI6WyJwdXNoIiwicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ0b2tlbiIsImVudGl0eV9yZWZlcmVuY2UiOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwia2luZCI6InRva2VuIiwidG9rZW4iOiJLMFZRWVlCWUhOWk9URElHNzhBM0FKOVBKVzVMM0gwVTIwU1NXMDJYRElUTDBQVkg1MzNNTklGU0JNNUdETDlYIiwiY29tLmFwb3N0aWxsZS5yb290cyI6eyJidWlsZF90ZXN0L3VwbG9hZF9maWxlX2ZpcmVmb3giOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.Oh4QJTCZRGgAJunRD4NOyu7UrY5Xn3bjZ2jJWQPKDl-VbM0hIpSusHP5SRDZp9T4SFfluMnIcCleEhdi7uW7933EJbHa-FRIHtD1FEPoaAOg9aFnnTd_WbAn3jWKa_rlbV9OWY7VTVeXAND1Em05Sk8mzMIRwQoWCckqWzAW7fIw0DJZljNRh6XcADz2t1BlfPoNslDvgK7Yv8JAeyW317D6cBAcbVfcI25gQ1Zbr9RM1BR9quLz9koUkTrZ-BXyGTIyM2anCy6thURDtfVNwqfD51MHxsDYkpw2j2RYzXSOtlvhavMs_VXw--eSEbHn3Z7D8f2Xnm_Gr6v9epqUyw gunicorn-registry stdout | 2025-11-04 09:33:29,350 [292] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:33:29,350 [292] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: None gunicorn-registry stdout | 2025-11-04 09:33:29,350 [292] [DEBUG] [auth.registry_jwt_auth] Identity changed to None gunicorn-registry stdout | 2025-11-04 09:33:29,350 [292] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'token', 'entity_reference': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'kind': 'token', 'token': 'K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U20SSW02XDITL0PVH533MNIFSBM5GDL9X', 'com.apostille.roots': {'build_test/upload_file_firefox': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:33:29,351 [292] [DEBUG] [endpoints.v2] Checking permission for repo: build_test/upload_file_firefox quotatotalworker stdout | 2025-11-04 09:33:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:33:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:44.351216+00:00 (in 14.999635 seconds) quotatotalworker stdout | 2025-11-04 09:33:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:44 GMT)" (scheduled at 2025-11-04 09:33:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:33:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size gunicorn-registry stdout | 2025-11-04 09:33:29,352 [292] [DEBUG] [data.cache.cache_key] Loading repository lookup from cache_key: repository_lookup_build_test_upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:29,352 [292] [DEBUG] [data.cache.impl] Checking cache for key repository_lookup_build_test_upload_file_firefox gunicorn-registry stdout | 2025-11-04 09:33:29,352 [292] [DEBUG] [data.cache.impl] Found no result in cache for key repository_lookup_build_test_upload_file_firefox; calling loader quotatotalworker stdout | 2025-11-04 09:33:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) gunicorn-registry stdout | 2025-11-04 09:33:29,353 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,356 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,359 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,362 [292] [DEBUG] [data.cache.impl] Got loaded result for key repository_lookup_build_test_upload_file_firefox: {'id': 2, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}} gunicorn-registry stdout | 2025-11-04 09:33:29,362 [292] [DEBUG] [data.cache.impl] Caching loaded result for key repository_lookup_build_test_upload_file_firefox with expiration {'id': 2, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:33:29,362 [292] [DEBUG] [data.cache.impl] Cached loaded result for key repository_lookup_build_test_upload_file_firefox with expiration {'id': 2, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:33:29,363 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."digest" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:d47233dabe62b1625e2a132b59c6b403d5e7c30c6d7441cc611890827b2bebf7', 1, 0]) quotatotalworker stdout | 2025-11-04 09:33:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:33:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:33:29,367 [292] [DEBUG] [peewee] ('(SELECT * FROM (SELECT "t1"."id", "t1"."content_checksum", "t1"."image_size", "t1"."uuid", "t1"."cas_path", "t1"."uncompressed_size" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s) AS "q0") UNION ALL ((SELECT * FROM (SELECT "t3"."id", "t3"."content_checksum", "t3"."image_size", "t3"."uuid", "t3"."cas_path", "t3"."uncompressed_size" FROM "imagestorage" AS "t3" INNER JOIN "uploadedblob" AS "t4" ON ("t4"."blob_id" = "t3"."id") WHERE (("t4"."repository_id" = %s) AND ("t3"."content_checksum" = %s)) LIMIT %s) AS "q1") UNION ALL (SELECT * FROM (SELECT "t5"."id", "t5"."content_checksum", "t5"."image_size", "t5"."uuid", "t5"."cas_path", "t5"."uncompressed_size" FROM "imagestorage" AS "t5" INNER JOIN "uploadedblob" AS "t6" ON ("t6"."blob_id" = "t5"."id") WHERE (("t6"."repository_id" = %s) AND ("t5"."content_checksum" = %s)) LIMIT %s) AS "q2"))', [2, 'sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 1, 2, 'sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 1, 2, 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,371 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."digest" = %s) AND ("t1"."repository_id" = %s)) LIMIT %s OFFSET %s', ['sha256:d47233dabe62b1625e2a132b59c6b403d5e7c30c6d7441cc611890827b2bebf7', 2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,375 [292] [DEBUG] [peewee] ('INSERT INTO "manifest" ("repository_id", "digest", "media_type_id", "manifest_bytes", "config_media_type", "layers_compressed_size", "subject", "subject_backfilled", "artifact_type", "artifact_type_backfilled") VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING "manifest"."id"', [2, 'sha256:d47233dabe62b1625e2a132b59c6b403d5e7c30c6d7441cc611890827b2bebf7', 17, '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8","size":7830},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5","size":80005019},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0","size":20144803}],"annotations":{"org.opencontainers.image.base.digest":"sha256:dce9f8caf5b4c3a3a511f71b0e2f23a6eada9bd1c8cc3c912f0e50b6d3b824fc","org.opencontainers.image.base.name":"registry.access.redhat.com/ubi7/ubi:latest"}}', 'application/vnd.oci.image.config.v1+json', 100149822, None, True, None, True]) gunicorn-registry stdout | 2025-11-04 09:33:29,377 [292] [DEBUG] [peewee] ('INSERT INTO "manifestblob" ("repository_id", "manifest_id", "blob_id") VALUES (%s, %s, %s), (%s, %s, %s), (%s, %s, %s) RETURNING "manifestblob"."id"', [2, 2, 2, 2, 2, 5, 2, 2, 6]) gunicorn-registry stdout | 2025-11-04 09:33:29,379 [292] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,380 [292] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [2, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,381 [292] [DEBUG] [peewee] ('SELECT 1 FROM "manifestblob" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ((("t2"."namespace_user_id" = %s) AND ("t1"."blob_id" = %s)) AND ("t1"."manifest_id" != %s)) LIMIT %s', [2, 5, 2, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,383 [292] [DEBUG] [peewee] ('SELECT 1 FROM "manifestblob" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ((("t2"."namespace_user_id" = %s) AND ("t1"."blob_id" = %s)) AND ("t1"."manifest_id" != %s)) LIMIT %s', [2, 2, 2, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,384 [292] [DEBUG] [peewee] ('SELECT 1 FROM "manifestblob" AS "t1" WHERE ((("t1"."repository_id" = %s) AND ("t1"."blob_id" = %s)) AND ("t1"."manifest_id" != %s)) LIMIT %s', [2, 2, 2, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,385 [292] [DEBUG] [peewee] ('SELECT 1 FROM "manifestblob" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ((("t2"."namespace_user_id" = %s) AND ("t1"."blob_id" = %s)) AND ("t1"."manifest_id" != %s)) LIMIT %s', [2, 6, 2, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,387 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,388 [292] [INFO] [data.model.quota] updating namespace size for manifest 2 in namespace 2, QuotaOperation.ADD 20152633 gunicorn-registry stdout | 2025-11-04 09:33:29,388 [292] [DEBUG] [peewee] ('UPDATE "quotanamespacesize" SET "size_bytes" = ("quotanamespacesize"."size_bytes" + %s) WHERE ("quotanamespacesize"."namespace_user_id" = %s)', [20152633, 2]) gunicorn-registry stdout | 2025-11-04 09:33:29,389 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotarepositorysize" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,391 [292] [INFO] [data.model.quota] updating repository size for manifest 2 in repository 2, QuotaOperation.ADD 20152633 gunicorn-registry stdout | 2025-11-04 09:33:29,391 [292] [DEBUG] [peewee] ('UPDATE "quotarepositorysize" SET "size_bytes" = ("quotarepositorysize"."size_bytes" + %s) WHERE ("quotarepositorysize"."repository_id" = %s)', [20152633, 2]) gunicorn-registry stdout | 2025-11-04 09:33:29,394 [292] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [2, 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,398 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['6dd31602-c2a2-4ba8-9b54-5e1279d3f1f7']) gunicorn-registry stdout | 2025-11-04 09:33:29,401 [292] [DEBUG] [boto3.resources.factory] Loading s3:Object gunicorn-registry stdout | 2025-11-04 09:33:29,401 [292] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7f679f49dd00> gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [boto3.resources.action] Calling s3:get_object with {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8'} gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-registry stdout | 2025-11-04 09:33:29,402 [292] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:33:29,403 [292] [DEBUG] [botocore.hooks] Event before-call.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,403 [292] [DEBUG] [botocore.hooks] Event before-call.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,403 [292] [DEBUG] [botocore.hooks] Event before-call.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,403 [292] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=GetObject) with params: {'url_path': '/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'query_string': {}, 'method': 'GET', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'url': 'https://s3.us-east-1.amazonaws.com/whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 's3v4', 's3_redirect': {'redirected': False, 'bucket': 'whuaws', 'params': {'Bucket': 'whuaws', 'Key': 'quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:33:29,403 [292] [DEBUG] [botocore.hooks] Event request-created.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,403 [292] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,403 [292] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | GET gunicorn-registry stdout | /whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 gunicorn-registry stdout | host:s3.us-east-1.amazonaws.com gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T093329Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T093329Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 50b7be7f30ed428d23133ad4875351c775b0cc7a1e70798fd5b16a58f4e2796c gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 5ef39a5ff34abf684045c5489037c94cb71f38eb7f3b3d5b38623abb5731cb5e gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.hooks] Event request-created.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.hooks] Event before-send.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T093329Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=AKIAXID22CKM25F5FFPX/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5ef39a5ff34abf684045c5489037c94cb71f38eb7f3b3d5b38623abb5731cb5e', 'amz-sdk-invocation-id': b'b076c586-5d9e-4886-b334-d612980eff70', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:33:29,404 [292] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:33:29,422 [292] [DEBUG] [urllib3.connectionpool] https://s3.us-east-1.amazonaws.com:443 "GET /whuaws/quaydata/sha256/11/11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8 HTTP/1.1" 200 7830 gunicorn-registry stdout | 2025-11-04 09:33:29,422 [292] [DEBUG] [botocore.parsers] Response headers: {'x-amz-id-2': '3XR7BbISrsUIUP3xREygjFBiu0LK7+mURCL27/9h2/2rqQE58xrKZIQfsGH7e5IT2/L1ZNigjoY=', 'x-amz-request-id': 'R4EW72ZSX9SE0WA2', 'Date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'Last-Modified': 'Tue, 04 Nov 2025 09:33:30 GMT', 'ETag': '"3a464cac103617123b1d463b51fbda26"', 'x-amz-server-side-encryption': 'AES256', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/octet-stream', 'Content-Length': '7830', 'Server': 'AmazonS3'} gunicorn-registry stdout | 2025-11-04 09:33:29,422 [292] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | gunicorn-registry stdout | 2025-11-04 09:33:29,423 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:33:29,423 [292] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:33:29,423 [292] [DEBUG] [botocore.hooks] Event needs-retry.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,423 [292] [DEBUG] [botocore.hooks] Event after-call.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:33:29,423 [292] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'R4EW72ZSX9SE0WA2', 'HostId': '3XR7BbISrsUIUP3xREygjFBiu0LK7+mURCL27/9h2/2rqQE58xrKZIQfsGH7e5IT2/L1ZNigjoY=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': '3XR7BbISrsUIUP3xREygjFBiu0LK7+mURCL27/9h2/2rqQE58xrKZIQfsGH7e5IT2/L1ZNigjoY=', 'x-amz-request-id': 'R4EW72ZSX9SE0WA2', 'date': 'Tue, 04 Nov 2025 09:33:30 GMT', 'last-modified': 'Tue, 04 Nov 2025 09:33:30 GMT', 'etag': '"3a464cac103617123b1d463b51fbda26"', 'x-amz-server-side-encryption': 'AES256', 'accept-ranges': 'bytes', 'content-type': 'application/octet-stream', 'content-length': '7830', 'server': 'AmazonS3'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 9, 33, 30, tzinfo=tzutc()), 'ContentLength': 7830, 'ETag': '"3a464cac103617123b1d463b51fbda26"', 'ContentType': 'application/octet-stream', 'ServerSideEncryption': 'AES256', 'Metadata': {}, 'Body': } gunicorn-registry stdout | 2025-11-04 09:33:29,425 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "mediatype" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:33:29,428 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."mutable" FROM "labelsourcetype" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:33:29,432 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,435 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['0a9fabae-aa38-49e7-9247-65686ee73105', 'architecture', 'x86_64', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,438 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 22]) gunicorn-registry stdout | 2025-11-04 09:33:29,442 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,445 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['a399835f-d446-4f97-aa76-cc19c39d71f9', 'build-date', '2024-05-29T03:00:15', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,447 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 23]) gunicorn-registry stdout | 2025-11-04 09:33:29,451 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,454 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['84c8848a-77d1-41fc-a154-911fffb8451f', 'com.redhat.component', 'ubi7-container', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,456 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 24]) gunicorn-registry stdout | 2025-11-04 09:33:29,461 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,464 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['1488bc28-a084-4baa-868e-aa8530911156', 'com.redhat.license_terms', 'https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,467 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 25]) gunicorn-registry stdout | 2025-11-04 09:33:29,471 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,474 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['486f1dda-3e02-4568-8f30-d0a758cb478d', 'description', 'The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,476 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 26]) gunicorn-registry stdout | 2025-11-04 09:33:29,481 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,484 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['7590ed45-8cab-4b5b-88ba-863a322daf68', 'distribution-scope', 'public', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,487 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 27]) gunicorn-registry stdout | 2025-11-04 09:33:29,491 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,495 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['6174a0ec-3547-481d-8258-134e6390a32b', 'io.buildah.version', '1.33.12', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,497 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 28]) gunicorn-registry stdout | 2025-11-04 09:33:29,501 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,504 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['44a5aa36-4fd6-4015-a4fe-6c304d28a033', 'io.k8s.description', 'The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,506 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 29]) gunicorn-registry stdout | 2025-11-04 09:33:29,509 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,513 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['582fc9c3-d96a-4a4f-8ae7-09eb91da3395', 'io.k8s.display-name', 'Red Hat Universal Base Image 7', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,515 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 30]) gunicorn-registry stdout | 2025-11-04 09:33:29,519 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,522 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['2f009bbf-4c0f-4d2f-99f7-ae6580c376a2', 'io.openshift.tags', 'base rhel7', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,524 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 31]) gunicorn-registry stdout | 2025-11-04 09:33:29,528 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,531 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['0b659015-6fd4-4ac3-807f-7ce29dd13528', 'maintainer', 'whu', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,533 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 32]) gunicorn-registry stdout | 2025-11-04 09:33:29,536 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,540 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['ed937fba-0bf9-4957-968c-37c8143c1123', 'name', 'ubi7', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,542 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 33]) gunicorn-registry stdout | 2025-11-04 09:33:29,545 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,549 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['7eded43c-867f-4c2a-846d-ea15442b101b', 'release', '1445', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,551 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 34]) gunicorn-registry stdout | 2025-11-04 09:33:29,554 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,558 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['c8278cf0-d8dd-47af-8a7e-a581ebaf0b13', 'summary', 'Provides the latest release of the Red Hat Universal Base Image 7.', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,560 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 35]) gunicorn-registry stdout | 2025-11-04 09:33:29,563 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,567 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['54129e36-9713-4126-95f0-94ee8d7d4bbf', 'url', 'https://access.redhat.com/containers/#/registry.access.redhat.com/ubi7/images/7.9-1445', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,569 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 36]) gunicorn-registry stdout | 2025-11-04 09:33:29,573 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,576 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['cbb3093f-584f-402e-8fcb-014134f124e9', 'vcs-ref', 'a4e710a688a6374670ecdd56637c3f683d11cbe3', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,578 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 37]) gunicorn-registry stdout | 2025-11-04 09:33:29,582 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,585 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['d9602b50-18ae-4f17-8dfe-c5449eeb2d87', 'vcs-type', 'git', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,587 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 38]) gunicorn-registry stdout | 2025-11-04 09:33:29,591 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,594 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['d3c5a5d2-ebde-496d-ab0d-c6d560b134ee', 'vendor', 'Red Hat, Inc.', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,596 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 39]) gunicorn-registry stdout | 2025-11-04 09:33:29,600 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,604 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['495bac89-f863-4082-87ba-1989655d8d86', 'version', '7.9', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,606 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 40]) gunicorn-registry stdout | 2025-11-04 09:33:29,609 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,613 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['8684d3eb-1f7a-426b-9d3a-2c918e91af83', 'org.opencontainers.image.base.digest', 'sha256:dce9f8caf5b4c3a3a511f71b0e2f23a6eada9bd1c8cc3c912f0e50b6d3b824fc', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,615 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 41]) gunicorn-registry stdout | 2025-11-04 09:33:29,618 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state" FROM "manifest" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,622 [292] [DEBUG] [peewee] ('INSERT INTO "label" ("uuid", "key", "value", "media_type_id", "source_type_id") VALUES (%s, %s, %s, %s, %s) RETURNING "label"."id"', ['fb0105b4-8cc4-46d0-b575-e558d6827eb4', 'org.opencontainers.image.base.name', 'registry.access.redhat.com/ubi7/ubi:latest', 1, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,624 [292] [DEBUG] [peewee] ('INSERT INTO "manifestlabel" ("repository_id", "manifest_id", "label_id") VALUES (%s, %s, %s) RETURNING "manifestlabel"."id"', [2, 2, 42]) gunicorn-registry stdout | 2025-11-04 09:33:29,627 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,630 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."size_bytes", "t1"."backfill_start_ms", "t1"."backfill_complete" FROM "quotanamespacesize" AS "t1" WHERE ("t1"."namespace_user_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,634 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."username" = %s) AND ("t1"."robot" = %s)) LIMIT %s OFFSET %s', ['build_test', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,637 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['build_test']) gunicorn-registry stdout | 2025-11-04 09:33:29,640 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."name" FROM "manifest" AS "t1" INNER JOIN "mediatype" AS "t2" ON ("t1"."media_type_id" = "t2"."id") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,644 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id", "t2"."id", "t2"."repository_id", "t2"."digest", "t2"."media_type_id", "t2"."manifest_bytes", "t2"."config_media_type", "t2"."layers_compressed_size", "t2"."subject", "t2"."subject_backfilled", "t2"."artifact_type", "t2"."artifact_type_backfilled" FROM "tag" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t1"."manifest_id" = "t2"."id") WHERE (((("t1"."repository_id" = %s) AND ("t1"."name" = %s)) AND (("t1"."lifetime_end_ms" IS %s) OR ("t1"."lifetime_end_ms" > %s))) AND ("t1"."hidden" = %s)) LIMIT %s OFFSET %s', [2, 'latest', None, 1762248809643, False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,646 [292] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [1]) gunicorn-registry stdout | 2025-11-04 09:33:29,648 [292] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: latest gunicorn-registry stdout | 2025-11-04 09:33:29,649 [292] [DEBUG] [peewee] ('UPDATE "tag" SET "lifetime_end_ms" = %s WHERE (("tag"."id" = %s) AND ("tag"."lifetime_end_ms" IS %s))', [1762248809643, 1, None]) gunicorn-registry stdout | 2025-11-04 09:33:29,650 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "tagkind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:33:29,651 [292] [DEBUG] [peewee] ('INSERT INTO "tag" ("name", "repository_id", "manifest_id", "lifetime_start_ms", "lifetime_end_ms", "immutable", "hidden", "reversion", "tag_kind_id") VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING "tag"."id"', ['latest', 2, 2, 1762248809643, None, False, False, False, 1]) gunicorn-registry stdout | 2025-11-04 09:33:29,656 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."friendly_name", "t1"."token_name", "t1"."token_code", "t1"."repository_id", "t1"."created", "t1"."role_id", "t1"."temporary", "t1"."kind_id", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed", "t4"."id", "t4"."name" FROM "accesstoken" AS "t1" INNER JOIN "role" AS "t4" ON ("t1"."role_id" = "t4"."id") INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ("t1"."token_name" = %s) LIMIT %s OFFSET %s', ['K0VQYYBYHNZOTDIG78A3AJ9PJW5L3H0U', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,659 [292] [DEBUG] [util.audit] Checking publishing push_repo to the user events system gunicorn-registry stdout | 2025-11-04 09:33:29,660 [292] [DEBUG] [util.audit] Logging the push_repo to analytics engine gunicorn-registry stdout | 2025-11-04 09:33:29,660 [292] [DEBUG] [util.audit] Resolving IP address 10.131.0.6 gunicorn-registry stdout | 2025-11-04 09:33:29,660 [292] [DEBUG] [util.audit] Resolved IP address 10.131.0.6 gunicorn-registry stdout | 2025-11-04 09:33:29,660 [292] [DEBUG] [util.audit] Logging the push_repo to logs system gunicorn-registry stdout | 2025-11-04 09:33:29,661 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['build_test', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:29,664 [292] [DEBUG] [peewee] ('INSERT INTO "logentry3" ("kind_id", "account_id", "performer_id", "repository_id", "datetime", "ip", "metadata_json") VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING "logentry3"."id"', [42, 2, None, 2, datetime.datetime(2025, 11, 4, 9, 33, 29, 661027), '10.131.0.6', '{"repo": "upload_file_firefox", "namespace": "build_test", "user-agent": "containers/5.29.5 (github.com/containers/image)", "tag": "latest", "token": "Repository Build Token", "resolved_ip": {"provider": "internet", "service": null, "sync_token": "1645662201", "country_iso_code": null, "aws_region": null, "continent": null}}']) gunicorn-registry stdout | 2025-11-04 09:33:29,668 [292] [DEBUG] [util.audit] Track and log of push_repo complete gunicorn-registry stdout | 2025-11-04 09:33:29,669 [292] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."event_id", "t1"."method_id", "t1"."title", "t1"."config_json", "t1"."event_config_json", "t1"."number_of_failures", "t1"."last_ran_ms", "t2"."id", "t2"."namespace_user_id", "t2"."name", "t2"."visibility_id", "t2"."description", "t2"."badge_token", "t2"."kind_id", "t2"."trust_enabled", "t2"."state", "t3"."id", "t3"."uuid", "t3"."username", "t3"."password_hash", "t3"."email", "t3"."verified", "t3"."stripe_id", "t3"."organization", "t3"."robot", "t3"."invoice_email", "t3"."invalid_login_attempts", "t3"."last_invalid_login", "t3"."removed_tag_expiration_s", "t3"."enabled", "t3"."invoice_email_address", "t3"."given_name", "t3"."family_name", "t3"."company", "t3"."location", "t3"."maximum_queued_builds_count", "t3"."creation_date", "t3"."last_accessed" FROM "repositorynotification" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") INNER JOIN "externalnotificationevent" AS "t4" ON ("t1"."event_id" = "t4"."id") WHERE ((("t3"."username" = %s) AND ("t2"."name" = %s)) AND ("t4"."name" = %s))', ['build_test', 'upload_file_firefox', 'repo_push']) gunicorn-registry stdout | 2025-11-04 09:33:29,673 [292] [DEBUG] [app] Ending request: urn:request:771cd083-c5aa-4589-9717-f1853377ed9a (/v2/build_test/upload_file_firefox/manifests/latest) {'endpoint': 'v2.write_manifest_by_tagname', 'request_id': 'urn:request:771cd083-c5aa-4589-9717-f1853377ed9a', 'remote_addr': '10.131.0.6', 'http_method': 'PUT', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/v2/build_test/upload_file_firefox/manifests/latest', 'path': '/v2/build_test/upload_file_firefox/manifests/latest', 'parameters': {}, 'json_body': {'schemaVersion': 2, 'mediaType': 'application/vnd.oci.image.manifest.v1+json', 'config': {'mediaType': 'application/vnd.oci.image.config.v1+json', 'digest': 'sha256:11e5d17f63057372f2e72afce164a64b062c9dd51ddff7cb3314c704d10135c8', 'size': 7830}, 'layers': [{'mediaType': 'application/vnd.oci.image.layer.v1.tar+gzip', 'digest': 'sha256:7f2c2c4492b6b2d181be862a0a1d1b6f6851cb07244efbcb43d44f9936aa78d5', 'size': 80005019}, {'mediaType': 'application/vnd.oci.image.layer.v1.tar+gzip', 'digest': 'sha256:03d1a8fc82a768fc6ec1cb02fbc03914f5e18636b12791a91eb13a4a07cb9fc0', 'size': 20144803}], 'annotations': {'org.opencontainers.image.base.digest': 'sha256:dce9f8caf5b4c3a3a511f71b0e2f23a6eada9bd1c8cc3c912f0e50b6d3b824fc', 'org.opencontainers.image.base.name': 'registry.access.redhat.com/ubi7/ubi:latest'}}, 'confsha': '76b27b45', 'user-agent': 'containers/5.29.5 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:33:29,673 [292] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:33:29,673 [292] [DEBUG] [peewee.pool] Returning 140083010304448 to pool. gunicorn-registry stdout | 2025-11-04 09:33:29,673 [292] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:29 +0000] "PUT /v2/build_test/upload_file_firefox/manifests/latest HTTP/1.1" 201 2 "-" "containers/5.29.5 (github.com/containers/image)" nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:29 +0000] "PUT /v2/build_test/upload_file_firefox/manifests/latest HTTP/1.1" 201 2 "-" "containers/5.29.5 (github.com/containers/image)" (0.325 2408 0.325) nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:29 +0000] "POST /buildman_pb.BuildManager/Heartbeat HTTP/2.0" 200 105 "-" "grpc-go/1.71.0" (29.923 14689 29.923) gunicorn-web stdout | 2025-11-04 09:33:29,687 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:29,697 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:29,713 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:29,713 [66] [DEBUG] [buildman.manager.ephemeral] Calling job complete callback for job buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff with result complete builder stdout | 2025-11-04 09:33:29,732 [66] [DEBUG] [data.database] Disconnecting from database. builder stdout | 2025-11-04 09:33:29,732 [66] [WARNING] [buildman.manager.ephemeral] Job buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff completed with result complete. Marking build done in queue. builder stdout | 2025-11-04 09:33:29,743 [66] [DEBUG] [buildman.manager.ephemeral] Terminating executor openshift with execution id 4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j builder stdout | 2025-11-04 09:33:29,743 [66] [DEBUG] [buildman.manager.executor] Executor openshift DEBUG set, not calling 'stop_builder()' builder stdout | 2025-11-04 09:33:29,744 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 1, "key": "buildman/production/lock/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": ""} builder stdout | 2025-11-04 09:33:29,744 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 3, "key": "buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"job_queue_item\": {\"id\": 3, \"body\": \"{\\\"build_uuid\\\": \\\"4708783e-59eb-4e42-b7a1-1923a4b4caff\\\", \\\"pull_credentials\\\": null}\", \"retries_remaining\": 2}, \"max_expiration\": 1762255975, \"last_heartbeat\": \"2025-11-04 09:33:27.909377\", \"created_at\": 1762248775.184402, \"executor_name\": \"openshift\", \"execution_id\": \"4708783e-59eb-4e42-b7a1-1923a4b4caff-nj86j\"}"} builder stdout | 2025-11-04 09:33:29,745 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 3, "key": "buildman/production/metric/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": "{\"executor_name\": \"openshift\", \"start_time\": 1762248775.3124962}"} builder stdout | 2025-11-04 09:33:29,745 [66] [DEBUG] [buildman.orchestrator] publishing event: {"event": 3, "key": "buildman/production/lock/4708783e-59eb-4e42-b7a1-1923a4b4caff", "value": ""} builder stdout | 2025-11-04 09:33:29,746 [66] [DEBUG] [buildman.manager.ephemeral] Job completed for job buildman/production/building/4708783e-59eb-4e42-b7a1-1923a4b4caff with result complete nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:29 +0000] "POST /buildman_pb.BuildManager/SetPhase HTTP/2.0" 200 9 "-" "grpc-go/1.71.0" (0.066 991 0.066) gunicorn-web stdout | 2025-11-04 09:33:29,957 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:30,143 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:30,155 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:30,247 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:30,697 [285] [DEBUG] [app] Starting request: urn:request:a5bf71fe-5cb5-4b3f-8abd-1ed381435849 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:30,701 [298] [DEBUG] [app] Starting request: urn:request:f2664b6f-ef43-45c9-8ef7-629d04e1c8ad (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:30,701 [298] [DEBUG] [app] Ending request: urn:request:f2664b6f-ef43-45c9-8ef7-629d04e1c8ad (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f2664b6f-ef43-45c9-8ef7-629d04e1c8ad', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:33:30,701 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:33:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:30,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:30,704 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:30,705 [287] [DEBUG] [app] Starting request: urn:request:41098016-2924-43e3-89c7-3ab371120e8e (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:30,706 [287] [DEBUG] [app] Ending request: urn:request:41098016-2924-43e3-89c7-3ab371120e8e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:41098016-2924-43e3-89c7-3ab371120e8e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:33:30,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:33:30,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:30,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:30,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:30,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:30,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:30,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:30,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:30,726 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:30,729 [285] [DEBUG] [app] Ending request: urn:request:a5bf71fe-5cb5-4b3f-8abd-1ed381435849 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a5bf71fe-5cb5-4b3f-8abd-1ed381435849', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:30,729 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:30,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:33:30,784 [287] [DEBUG] [app] Starting request: urn:request:ba4656f8-51ef-4c34-8079-7864a9a004c8 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:30,784 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:30,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:30,788 [299] [DEBUG] [app] Starting request: urn:request:8fdec159-3b50-4b2a-a831-2e1f73eee326 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:30,788 [299] [DEBUG] [app] Ending request: urn:request:8fdec159-3b50-4b2a-a831-2e1f73eee326 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8fdec159-3b50-4b2a-a831-2e1f73eee326', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:33:30,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:33:30,788 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:30,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:30,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:30,792 [285] [DEBUG] [app] Starting request: urn:request:f372b640-43f0-426e-86f3-169d007f7db3 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:30,792 [285] [DEBUG] [app] Ending request: urn:request:f372b640-43f0-426e-86f3-169d007f7db3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f372b640-43f0-426e-86f3-169d007f7db3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:33:30,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:33:30,792 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:30,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:30,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:30,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:30,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:30,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:30,809 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:30,812 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:30,815 [287] [DEBUG] [app] Ending request: urn:request:ba4656f8-51ef-4c34-8079-7864a9a004c8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ba4656f8-51ef-4c34-8079-7864a9a004c8', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:30,815 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:30,816 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.032) nginx stdout | 10.129.2.10 (-) - - [04/Nov/2025:09:33:31 +0000] "POST /buildman_pb.BuildManager/LogMessage HTTP/2.0" 200 3193 "-" "grpc-go/1.71.0" (31.411 402052 31.411) notificationworker stdout | 2025-11-04 09:33:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:33:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:41.530064+00:00 (in 9.999537 seconds) notificationworker stdout | 2025-11-04 09:33:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:41 GMT)" (scheduled at 2025-11-04 09:33:31.530064+00:00) notificationworker stdout | 2025-11-04 09:33:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:33:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 31, 530790), True, datetime.datetime(2025, 11, 4, 9, 33, 31, 530790), 0, 'notification/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:31,533 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:33:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:33:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:33:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:41 GMT)" executed successfully gcworker stdout | 2025-11-04 09:33:32,329 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:33:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:33:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:21.434460+00:00 (in 48.998251 seconds) namespacegcworker stdout | 2025-11-04 09:33:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:32 GMT)" (scheduled at 2025-11-04 09:33:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:33:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:33:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:32 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:33:32,526 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:33:32,540 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:33:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:33:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:48.625192+00:00 (in 15.001809 seconds) exportactionlogsworker stdout | 2025-11-04 09:33:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:33 GMT)" (scheduled at 2025-11-04 09:33:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:33:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:33:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 33, 623682), True, datetime.datetime(2025, 11, 4, 9, 33, 33, 623682), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:33:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:33:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:33:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:33:33,817 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:33:33,817 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:33:33,921 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:34,157 [285] [DEBUG] [app] Starting request: urn:request:a88c980b-1664-41e6-b8e8-26521ff51562 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'X-Forwarded-For': '10.131.0.6'} gunicorn-web stdout | 2025-11-04 09:33:34,158 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,158 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,158 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,170 [285] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,171 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:34,176 [285] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,177 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:34,182 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:34,186 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:34,191 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,191 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'whuaws', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.us-east-1.amazonaws.com', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.us-east-1.amazonaws.com/whuaws gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "s3v4" gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.regions] Selected auth type "v4" as "s3v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:34,192 [285] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-web stdout | 2025-11-04 09:33:34,193 [285] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-web stdout | 2025-11-04 09:33:34,193 [285] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-web stdout | GET gunicorn-web stdout | /whuaws/quaydata/userfiles/3b4ed7ae-c4a4-44f4-b856-4499356fd36b gunicorn-web stdout | X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAXID22CKM25F5FFPX%2F20251104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251104T093334Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host gunicorn-web stdout | host:s3.us-east-1.amazonaws.com gunicorn-web stdout | host gunicorn-web stdout | UNSIGNED-PAYLOAD gunicorn-web stdout | 2025-11-04 09:33:34,193 [285] [DEBUG] [botocore.auth] StringToSign: gunicorn-web stdout | AWS4-HMAC-SHA256 gunicorn-web stdout | 20251104T093334Z gunicorn-web stdout | 20251104/us-east-1/s3/aws4_request gunicorn-web stdout | 5da77eac5a8471cc73906b9c506cf20af13896161f3baaaa4a8a354c84c2f8b4 gunicorn-web stdout | 2025-11-04 09:33:34,193 [285] [DEBUG] [botocore.auth] Signature: gunicorn-web stdout | 00d2b671f12e7229e34b446986c1ba63c2b63a2507965cef23d1e5f562112012 gunicorn-web stdout | 2025-11-04 09:33:34,193 [285] [DEBUG] [app] Ending request: urn:request:a88c980b-1664-41e6-b8e8-26521ff51562 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff) {'endpoint': 'api.repositorybuildresource', 'request_id': 'urn:request:a88c980b-1664-41e6-b8e8-26521ff51562', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:33:34,193 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:34 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/2.0" 200 889 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.038 1027 0.037) gunicorn-web stdout | 2025-11-04 09:33:34,194 [285] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:34 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff HTTP/1.0" 200 889 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" gunicorn-web stdout | 2025-11-04 09:33:34,411 [285] [DEBUG] [app] Starting request: urn:request:1d23a478-5854-4bec-9644-c190089c1598 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'X-Forwarded-For': '10.131.0.6'} gunicorn-web stdout | 2025-11-04 09:33:34,411 [285] [DEBUG] [app] User loader loading deferred user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,411 [285] [DEBUG] [auth.cookie] Loading user from cookie: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,412 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['b6b28aab-96ae-472a-887e-d876bbda1207', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:34,423 [285] [DEBUG] [auth.decorators] Found valid auth result: (, False, None, [, None, None, None, None, None, None]) gunicorn-web stdout | 2025-11-04 09:33:34,423 [285] [DEBUG] [auth.permissions] Identity loaded: gunicorn-web stdout | 2025-11-04 09:33:34,423 [285] [DEBUG] [auth.permissions] Deferring permissions for user with uuid: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,424 [285] [DEBUG] [endpoints.api] Checking permission for repo: build_test/upload_file_firefox gunicorn-web stdout | 2025-11-04 09:33:34,424 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,424 [285] [DEBUG] [auth.permissions] User permission: _UserTypeNeed(type='user', username='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,424 [285] [DEBUG] [auth.permissions] User namespace permission: _NamespaceWideNeed(type='organization', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,424 [285] [DEBUG] [auth.permissions] User namespace repo permission: _NamespaceWideNeed(type='organizationrepo', namespace='whuquay', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,425 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t1"."user_id" = "t5"."id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t5"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:34,429 [285] [DEBUG] [auth.permissions] User added permission: _ResourceNeed(type='repository', namespace='build_test', name='upload_file_firefox', role='admin') gunicorn-web stdout | 2025-11-04 09:33:34,430 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."team_id", "t1"."user_id", "t1"."repository_id", "t1"."role_id", "t2"."id", "t2"."name", "t3"."id", "t3"."namespace_user_id", "t3"."name", "t3"."visibility_id", "t3"."description", "t3"."badge_token", "t3"."kind_id", "t3"."trust_enabled", "t3"."state", "t4"."id", "t4"."uuid", "t4"."username", "t4"."password_hash", "t4"."email", "t4"."verified", "t4"."stripe_id", "t4"."organization", "t4"."robot", "t4"."invoice_email", "t4"."invalid_login_attempts", "t4"."last_invalid_login", "t4"."removed_tag_expiration_s", "t4"."enabled", "t4"."invoice_email_address", "t4"."given_name", "t4"."family_name", "t4"."company", "t4"."location", "t4"."maximum_queued_builds_count", "t4"."creation_date", "t4"."last_accessed" FROM "repositorypermission" AS "t1" INNER JOIN "role" AS "t2" ON ("t1"."role_id" = "t2"."id") INNER JOIN "repository" AS "t3" ON ("t1"."repository_id" = "t3"."id") INNER JOIN "user" AS "t4" ON ("t3"."namespace_user_id" = "t4"."id") INNER JOIN "team" AS "t5" ON ("t1"."team_id" = "t5"."id") INNER JOIN "teammember" AS "t6" ON ("t6"."team_id" = "t5"."id") INNER JOIN "user" AS "t7" ON ("t7"."id" = "t6"."user_id") WHERE ((("t3"."name" = %s) AND ("t4"."username" = %s)) AND ("t7"."id" = %s))', ['upload_file_firefox', 'build_test', 1]) gunicorn-web stdout | 2025-11-04 09:33:34,435 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") INNER JOIN "repositorykind" AS "t3" ON ("t1"."kind_id" = "t3"."id") WHERE (((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) AND ("t3"."name" = %s)) LIMIT %s OFFSET %s', ['build_test', 'upload_file_firefox', 3, 'application', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:34,439 [285] [DEBUG] [auth.permissions] Loading user permissions after deferring for: b6b28aab-96ae-472a-887e-d876bbda1207 gunicorn-web stdout | 2025-11-04 09:33:34,440 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id", "t2"."id", "t2"."uuid", "t2"."service_id", "t2"."repository_id", "t2"."connected_user_id", "t2"."secure_auth_token", "t2"."secure_private_key", "t2"."fully_migrated", "t2"."config", "t2"."write_token_id", "t2"."pull_robot_id", "t2"."enabled", "t2"."disabled_reason_id", "t2"."disabled_datetime", "t2"."successive_failure_count", "t2"."successive_internal_error_count", "t3"."id", "t3"."name", "t4"."id", "t4"."namespace_user_id", "t4"."name", "t4"."visibility_id", "t4"."description", "t4"."badge_token", "t4"."kind_id", "t4"."trust_enabled", "t4"."state", "t5"."id", "t5"."uuid", "t5"."username", "t5"."password_hash", "t5"."email", "t5"."verified", "t5"."stripe_id", "t5"."organization", "t5"."robot", "t5"."invoice_email", "t5"."invalid_login_attempts", "t5"."last_invalid_login", "t5"."removed_tag_expiration_s", "t5"."enabled", "t5"."invoice_email_address", "t5"."given_name", "t5"."family_name", "t5"."company", "t5"."location", "t5"."maximum_queued_builds_count", "t5"."creation_date", "t5"."last_accessed", "t6"."id", "t6"."uuid", "t6"."username", "t6"."password_hash", "t6"."email", "t6"."verified", "t6"."stripe_id", "t6"."organization", "t6"."robot", "t6"."invoice_email", "t6"."invalid_login_attempts", "t6"."last_invalid_login", "t6"."removed_tag_expiration_s", "t6"."enabled", "t6"."invoice_email_address", "t6"."given_name", "t6"."family_name", "t6"."company", "t6"."location", "t6"."maximum_queued_builds_count", "t6"."creation_date", "t6"."last_accessed" FROM "repositorybuild" AS "t1" INNER JOIN "repository" AS "t4" ON ("t1"."repository_id" = "t4"."id") INNER JOIN "user" AS "t5" ON ("t4"."namespace_user_id" = "t5"."id") LEFT OUTER JOIN "user" AS "t6" ON ("t1"."pull_robot_id" = "t6"."id") LEFT OUTER JOIN "repositorybuildtrigger" AS "t2" ON ("t1"."trigger_id" = "t2"."id") LEFT OUTER JOIN "buildtriggerservice" AS "t3" ON ("t2"."service_id" = "t3"."id") WHERE ("t1"."uuid" = %s) ORDER BY "t1"."started" DESC LIMIT %s OFFSET %s', ['4708783e-59eb-4e42-b7a1-1923a4b4caff', 1, 0]) gunicorn-web stdout | 2025-11-04 09:33:34,446 [285] [DEBUG] [app] Ending request: urn:request:1d23a478-5854-4bec-9644-c190089c1598 (/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs) {'endpoint': 'api.repositorybuildlogs', 'request_id': 'urn:request:1d23a478-5854-4bec-9644-c190089c1598', 'remote_addr': '10.131.0.6', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=317', 'path': '/api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs', 'parameters': {'start': '317'}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:33:34,446 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.6 (-) - - [04/Nov/2025:09:33:34 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=317 HTTP/2.0" 200 3347 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.037 1037 0.036) gunicorn-web stdout | 2025-11-04 09:33:34,447 [285] [INFO] [gunicorn.access] 10.131.0.6 - - [04/Nov/2025:09:33:34 +0000] "GET /api/v1/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff/logs?start=317 HTTP/1.0" 200 3347 "https://quayregistry-quay-quay-enterprise.apps.whu419az06.qe.azure.devcluster.openshift.com/repository/build_test/upload_file_firefox/build/4708783e-59eb-4e42-b7a1-1923a4b4caff" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" builder stdout | 2025-11-04 09:33:34,692 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:33:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:33:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:38.437104+00:00 (in 3.000894 seconds) gcworker stdout | 2025-11-04 09:33:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:05 GMT)" (scheduled at 2025-11-04 09:33:35.435809+00:00) gcworker stdout | 2025-11-04 09:33:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039215436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:33:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:33:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:33:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:33:36,007 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:33:36,376 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:33:36,996 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:33:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:33:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:40.124445+00:00 (in 3.001199 seconds) proxycacheblobworker stdout | 2025-11-04 09:33:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:47 GMT)" (scheduled at 2025-11-04 09:33:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:33:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:33:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 37, 123564), True, datetime.datetime(2025, 11, 4, 9, 33, 37, 123564), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:33:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:33:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:33:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:33:37,418 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:37,737 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:33:37,750 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:33:37,750 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:33:37,760 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:33:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:33:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:05.435809+00:00 (in 26.998260 seconds) gcworker stdout | 2025-11-04 09:33:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:08 GMT)" (scheduled at 2025-11-04 09:33:38.437104+00:00) gcworker stdout | 2025-11-04 09:33:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:33:38,442 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:38,446 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:33:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248758449, None, 1, 0]) gcworker stdout | 2025-11-04 09:33:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:33:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:33:38,454 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:38,466 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:38,494 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:38,505 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:38,535 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:38,696 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:33:38,699 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:33:38,843 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:33:38,845 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:33:38,847 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:33:39,546 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:33:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:33:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:01.922827+00:00 (in 21.997750 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:33:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:39 GMT)" (scheduled at 2025-11-04 09:33:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:33:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:33:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:33:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:33:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:47.122849+00:00 (in 6.998007 seconds) proxycacheblobworker stdout | 2025-11-04 09:33:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:40 GMT)" (scheduled at 2025-11-04 09:33:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:33:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:33:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:33:41,313 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:33:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:33:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:51.530064+00:00 (in 9.999612 seconds) notificationworker stdout | 2025-11-04 09:33:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:51 GMT)" (scheduled at 2025-11-04 09:33:41.530064+00:00) notificationworker stdout | 2025-11-04 09:33:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:33:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 41, 530778), True, datetime.datetime(2025, 11, 4, 9, 33, 41, 530778), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:33:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:33:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:33:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:33:42,765 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:33:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:33:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:57.523915+00:00 (in 14.090676 seconds) repositorygcworker stdout | 2025-11-04 09:33:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:43 GMT)" (scheduled at 2025-11-04 09:33:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:33:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:33:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 43, 433472), True, datetime.datetime(2025, 11, 4, 9, 33, 43, 433472), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:33:43,445 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:33:43,445 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:33:43,445 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:33:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:33:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:59.351216+00:00 (in 14.999623 seconds) quotatotalworker stdout | 2025-11-04 09:33:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:59 GMT)" (scheduled at 2025-11-04 09:33:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:33:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:33:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:33:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:33:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:33:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:33:45,271 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:33:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:33:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:47.479080+00:00 (in 1.998905 seconds) securityworker stdout | 2025-11-04 09:33:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:15 GMT)" (scheduled at 2025-11-04 09:33:45.479769+00:00) securityworker stdout | 2025-11-04 09:33:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:33:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:33:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:33:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:33:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:33:45,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 28, 45, 483952), 1, 3]) securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:33:45,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 28, 45, 483952), 1, 3]) securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:33:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:33:45,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:33:45,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:33:45,696 [285] [DEBUG] [app] Starting request: urn:request:571212f2-d2ac-4edf-9097-606d71b5415e (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:45,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:45,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:45,701 [292] [DEBUG] [app] Starting request: urn:request:2c59506b-752e-4539-a277-4856d35a15a4 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:45,701 [292] [DEBUG] [app] Ending request: urn:request:2c59506b-752e-4539-a277-4856d35a15a4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2c59506b-752e-4539-a277-4856d35a15a4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:33:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:33:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:45,706 [288] [DEBUG] [app] Starting request: urn:request:673bafaf-a5cb-426c-b099-2a7bcf83b9c9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:45,706 [288] [DEBUG] [app] Ending request: urn:request:673bafaf-a5cb-426c-b099-2a7bcf83b9c9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:673bafaf-a5cb-426c-b099-2a7bcf83b9c9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:33:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:33:45,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:45,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:45,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:45,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:45,731 [285] [DEBUG] [app] Ending request: urn:request:571212f2-d2ac-4edf-9097-606d71b5415e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:571212f2-d2ac-4edf-9097-606d71b5415e', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:45,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.035) gunicorn-web stdout | 2025-11-04 09:33:45,783 [285] [DEBUG] [app] Starting request: urn:request:c425eb7f-d891-44eb-a862-99f109f2c610 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:33:45,784 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:33:45,785 [292] [DEBUG] [app] Starting request: urn:request:cda46608-3280-4e75-b0fa-1c4e90bb3c9e (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:33:45,785 [292] [DEBUG] [app] Ending request: urn:request:cda46608-3280-4e75-b0fa-1c4e90bb3c9e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cda46608-3280-4e75-b0fa-1c4e90bb3c9e', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:33:45,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:33:45,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:45,786 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:33:45,788 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:33:45,789 [286] [DEBUG] [app] Starting request: urn:request:92dc5a2b-4399-4233-bd3e-f5d785f4b467 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:33:45,790 [286] [DEBUG] [app] Ending request: urn:request:92dc5a2b-4399-4233-bd3e-f5d785f4b467 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:92dc5a2b-4399-4233-bd3e-f5d785f4b467', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:33:45,790 [286] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:33:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:33:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:33:45,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:33:45,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:33:45,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:33:45,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:33:45,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:33:45,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:33:45,807 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:33:45,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:33:45,814 [285] [DEBUG] [app] Ending request: urn:request:c425eb7f-d891-44eb-a862-99f109f2c610 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c425eb7f-d891-44eb-a862-99f109f2c610', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:33:45,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:33:45,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:33:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:33:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) proxycacheblobworker stdout | 2025-11-04 09:33:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:33:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:33:57.122849+00:00 (in 9.999614 seconds) proxycacheblobworker stdout | 2025-11-04 09:33:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:57 GMT)" (scheduled at 2025-11-04 09:33:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:33:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:33:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 47, 123522), True, datetime.datetime(2025, 11, 4, 9, 33, 47, 123522), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:33:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:33:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:33:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:33:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:33:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:33:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:15.479769+00:00 (in 28.000194 seconds) securityworker stdout | 2025-11-04 09:33:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:17 GMT)" (scheduled at 2025-11-04 09:33:47.479080+00:00) securityworker stdout | 2025-11-04 09:33:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:33:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:33:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:33:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:33:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:33:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:47,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:33:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:33:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 28, 47, 483602), 1, 3]) securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:33:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:33:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 28, 47, 483602), 1, 3]) securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:33:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:33:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:33:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:17 GMT)" executed successfully builder stdout | 2025-11-04 09:33:47,760 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:33:47,773 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:33:47,773 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:33:47,783 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:33:48,207 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:33:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:33:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:33.622985+00:00 (in 44.997387 seconds) exportactionlogsworker stdout | 2025-11-04 09:33:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:48 GMT)" (scheduled at 2025-11-04 09:33:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:33:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:33:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:33:48,880 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:33:50,189 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:33:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:33:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:21.226294+00:00 (in 29.999531 seconds) autopruneworker stdout | 2025-11-04 09:33:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:21 GMT)" (scheduled at 2025-11-04 09:33:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:33:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248771234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:33:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:33:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:33:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:33:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:33:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:01.530064+00:00 (in 9.999580 seconds) notificationworker stdout | 2025-11-04 09:33:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:01 GMT)" (scheduled at 2025-11-04 09:33:51.530064+00:00) notificationworker stdout | 2025-11-04 09:33:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:33:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 51, 530768), True, datetime.datetime(2025, 11, 4, 9, 33, 51, 530768), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:33:51,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:33:51,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:33:51,542 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:33:52,468 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:33:55,315 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:33:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:33:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:07.122849+00:00 (in 9.999575 seconds) proxycacheblobworker stdout | 2025-11-04 09:33:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:07 GMT)" (scheduled at 2025-11-04 09:33:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:33:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:33:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 33, 57, 123573), True, datetime.datetime(2025, 11, 4, 9, 33, 57, 123573), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:33:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:09.133645+00:00 (in 12.001491 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:57 GMT)" (scheduled at 2025-11-04 09:33:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:33:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:33:57,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:33:57,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:33:57,135 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:33:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:33:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:33:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:33:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:33:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:27.222846+00:00 (in 29.999570 seconds) buildlogsarchiver stdout | 2025-11-04 09:33:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:27 GMT)" (scheduled at 2025-11-04 09:33:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:33:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 33, 57, 223469), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:33:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:33:57,238 [67] [DEBUG] [__main__] Archiving: 4708783e-59eb-4e42-b7a1-1923a4b4caff buildlogsarchiver stdout | 2025-11-04 09:33:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:33:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:33:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:33:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:33:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:02.523412+00:00 (in 4.999073 seconds) repositorygcworker stdout | 2025-11-04 09:33:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:57 GMT)" (scheduled at 2025-11-04 09:33:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:33:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:33:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:34:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:33:57,538 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:33:57,783 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:33:57,797 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:33:57,797 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:33:57,807 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:33:58,260 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:33:58,912 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:33:59,295 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:33:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:33:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:14.351216+00:00 (in 14.999567 seconds) quotatotalworker stdout | 2025-11-04 09:33:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:14 GMT)" (scheduled at 2025-11-04 09:33:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:33:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:33:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:33:59,366 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:33:59,366 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:33:59,698 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:33:59,972 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:00,158 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:00,170 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:00,261 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:00,697 [288] [DEBUG] [app] Starting request: urn:request:c68addcb-e99f-4883-ad10-ef82b53df362 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:00,701 [297] [DEBUG] [app] Starting request: urn:request:7b62e8f8-0b62-4fff-b1ef-eedf44fddbe8 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:00,702 [297] [DEBUG] [app] Ending request: urn:request:7b62e8f8-0b62-4fff-b1ef-eedf44fddbe8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7b62e8f8-0b62-4fff-b1ef-eedf44fddbe8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:34:00,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:34:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:00,706 [285] [DEBUG] [app] Starting request: urn:request:947cbd32-ff5c-4fe4-bf7d-b357a62e7dd4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:00,706 [285] [DEBUG] [app] Ending request: urn:request:947cbd32-ff5c-4fe4-bf7d-b357a62e7dd4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:947cbd32-ff5c-4fe4-bf7d-b357a62e7dd4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:34:00,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:00,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:00,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:00,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:00,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:00,730 [288] [DEBUG] [app] Ending request: urn:request:c68addcb-e99f-4883-ad10-ef82b53df362 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c68addcb-e99f-4883-ad10-ef82b53df362', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:00,730 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:34:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:34:00,784 [285] [DEBUG] [app] Starting request: urn:request:761b80f7-4e4a-4521-a0fc-0e5ead906c03 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:00,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:00,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:00,787 [299] [DEBUG] [app] Starting request: urn:request:f4e40c7d-17eb-47c4-822a-e5499332ed33 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:00,788 [299] [DEBUG] [app] Ending request: urn:request:f4e40c7d-17eb-47c4-822a-e5499332ed33 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f4e40c7d-17eb-47c4-822a-e5499332ed33', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:34:00,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:34:00,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:00,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:00,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:00,792 [287] [DEBUG] [app] Starting request: urn:request:8214d831-5e88-44b6-aaf2-6142b1e431d1 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:00,792 [287] [DEBUG] [app] Ending request: urn:request:8214d831-5e88-44b6-aaf2-6142b1e431d1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8214d831-5e88-44b6-aaf2-6142b1e431d1', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:34:00,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:00,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:00,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:00,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:00,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:00,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:00,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:00,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:00,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:00,816 [285] [DEBUG] [app] Ending request: urn:request:761b80f7-4e4a-4521-a0fc-0e5ead906c03 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:761b80f7-4e4a-4521-a0fc-0e5ead906c03', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:00,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:34:00,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) notificationworker stdout | 2025-11-04 09:34:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:11.530064+00:00 (in 9.999584 seconds) notificationworker stdout | 2025-11-04 09:34:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:11 GMT)" (scheduled at 2025-11-04 09:34:01.530064+00:00) notificationworker stdout | 2025-11-04 09:34:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:34:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 1, 530802), True, datetime.datetime(2025, 11, 4, 9, 34, 1, 530802), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:34:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:34:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:34:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:11 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:34:01,542 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:34:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:34:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:39.924652+00:00 (in 38.001424 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:34:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:01 GMT)" (scheduled at 2025-11-04 09:34:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:34:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:34:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 1, 923467), True, datetime.datetime(2025, 11, 4, 9, 34, 1, 923467), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:34:01,934 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:34:01,934 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:34:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:34:02,338 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:34:02,523 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:34:02,523 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:43.432867+00:00 (in 40.909026 seconds) repositorygcworker stdout | 2025-11-04 09:34:02,523 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:39:02 GMT)" (scheduled at 2025-11-04 09:34:02.523412+00:00) repositorygcworker stdout | 2025-11-04 09:34:02,524 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 34, 2, 524108), 'repositorygc/%']) repositoryactioncounter stdout | 2025-11-04 09:34:02,535 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:34:02,536 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 34, 2, 524108), True, datetime.datetime(2025, 11, 4, 9, 34, 2, 524108), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:34:02,540 [100] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 34, 2, 524108), True, datetime.datetime(2025, 11, 4, 9, 34, 2, 524108), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 34, 2, 524108), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:34:02,543 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:34:02,543 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:39:02 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:34:02,549 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:34:03,825 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:34:03,825 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:34:03,929 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:34:04,701 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:34:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:34:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:08.437104+00:00 (in 3.000909 seconds) gcworker stdout | 2025-11-04 09:34:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:35 GMT)" (scheduled at 2025-11-04 09:34:05.435809+00:00) gcworker stdout | 2025-11-04 09:34:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039245436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:34:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:34:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:34:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:34:06,016 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:34:06,384 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:34:07,005 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:34:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:17.122849+00:00 (in 9.999568 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:17 GMT)" (scheduled at 2025-11-04 09:34:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:34:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 7, 123690), True, datetime.datetime(2025, 11, 4, 9, 34, 7, 123690), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:34:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:34:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:34:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:34:07,440 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:34:07,807 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:34:07,821 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:34:07,821 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:34:07,831 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:34:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:34:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:35.435809+00:00 (in 26.998326 seconds) gcworker stdout | 2025-11-04 09:34:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:38 GMT)" (scheduled at 2025-11-04 09:34:08.437104+00:00) gcworker stdout | 2025-11-04 09:34:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:34:08,448 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248788448, None, 1, 0]) gcworker stdout | 2025-11-04 09:34:08,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:34:08,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:34:08,453 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,459 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,466 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,481 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,507 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,520 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,546 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,711 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:08,715 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:34:08,853 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:34:08,855 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:34:08,857 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:34:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:34:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:57.131795+00:00 (in 47.997775 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:34:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:09 GMT)" (scheduled at 2025-11-04 09:34:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:34:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:34:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:34:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:34:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:34:09,555 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:34:11,332 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:34:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:15.535095+00:00 (in 4.004572 seconds) notificationworker stdout | 2025-11-04 09:34:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:21 GMT)" (scheduled at 2025-11-04 09:34:11.530064+00:00) notificationworker stdout | 2025-11-04 09:34:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:34:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 11, 530803), True, datetime.datetime(2025, 11, 4, 9, 34, 11, 530803), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:34:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:34:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:34:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:34:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:34:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:12.222828+00:00 (in 59.999580 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:34:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:12 GMT)" (scheduled at 2025-11-04 09:34:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:34:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:34:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:34:12,235 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:34:12,780 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:34:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:34:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:29.351216+00:00 (in 14.999599 seconds) quotatotalworker stdout | 2025-11-04 09:34:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:29 GMT)" (scheduled at 2025-11-04 09:34:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:34:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:34:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:34:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:34:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:34:15,281 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:34:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:34:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:17.479080+00:00 (in 1.998880 seconds) securityworker stdout | 2025-11-04 09:34:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:45 GMT)" (scheduled at 2025-11-04 09:34:15.479769+00:00) securityworker stdout | 2025-11-04 09:34:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:34:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:34:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:34:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:15,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 29, 15, 484103), 1, 3]) securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 29, 15, 484103), 1, 3]) securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:34:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:34:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:34:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:21.530064+00:00 (in 5.994522 seconds) notificationworker stdout | 2025-11-04 09:34:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:15 GMT)" (scheduled at 2025-11-04 09:34:15.535095+00:00) notificationworker stdout | 2025-11-04 09:34:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:34:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:34:15,697 [288] [DEBUG] [app] Starting request: urn:request:af452751-a499-40a4-ad2b-21abddb31815 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:15,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:15,701 [297] [DEBUG] [app] Starting request: urn:request:7e25f80b-abfd-4246-9efe-30be0e9fb10f (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:15,701 [297] [DEBUG] [app] Ending request: urn:request:7e25f80b-abfd-4246-9efe-30be0e9fb10f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7e25f80b-abfd-4246-9efe-30be0e9fb10f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:34:15,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:34:15,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:15,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:15,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:15,706 [285] [DEBUG] [app] Starting request: urn:request:818530be-4e5d-4c3a-b1ef-c9f4cf1c4f4a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:15,706 [285] [DEBUG] [app] Ending request: urn:request:818530be-4e5d-4c3a-b1ef-c9f4cf1c4f4a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:818530be-4e5d-4c3a-b1ef-c9f4cf1c4f4a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:34:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:15,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:15,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:15,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:15,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:15,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:15,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:15,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:15,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:15,730 [288] [DEBUG] [app] Ending request: urn:request:af452751-a499-40a4-ad2b-21abddb31815 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:af452751-a499-40a4-ad2b-21abddb31815', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:15,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:34:15,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:34:15,784 [287] [DEBUG] [app] Starting request: urn:request:edc6501c-a6b5-48ba-a1f2-8ba490851d50 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:15,784 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:15,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:15,788 [299] [DEBUG] [app] Starting request: urn:request:77dc6729-c4b0-48b9-bf23-2a0eea281351 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:15,788 [299] [DEBUG] [app] Ending request: urn:request:77dc6729-c4b0-48b9-bf23-2a0eea281351 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:77dc6729-c4b0-48b9-bf23-2a0eea281351', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:34:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:34:15,788 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:15,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:15,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:15,792 [285] [DEBUG] [app] Starting request: urn:request:6cbb5028-4d04-4597-bd61-aec50eff9d3d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:15,792 [285] [DEBUG] [app] Ending request: urn:request:6cbb5028-4d04-4597-bd61-aec50eff9d3d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6cbb5028-4d04-4597-bd61-aec50eff9d3d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:34:15,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:15,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:15,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:15,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:15,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:15,800 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:15,800 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:15,810 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:15,813 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:15,816 [287] [DEBUG] [app] Ending request: urn:request:edc6501c-a6b5-48ba-a1f2-8ba490851d50 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:edc6501c-a6b5-48ba-a1f2-8ba490851d50', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:15,816 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:34:15,816 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:34:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:27.122849+00:00 (in 9.999592 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:27 GMT)" (scheduled at 2025-11-04 09:34:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:34:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 17, 123520), True, datetime.datetime(2025, 11, 4, 9, 34, 17, 123520), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:34:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:34:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:34:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:34:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:34:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:45.479769+00:00 (in 28.000220 seconds) securityworker stdout | 2025-11-04 09:34:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:47 GMT)" (scheduled at 2025-11-04 09:34:17.479080+00:00) securityworker stdout | 2025-11-04 09:34:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:34:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:34:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:34:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:34:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:34:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:47 GMT)" executed successfully builder stdout | 2025-11-04 09:34:17,831 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:34:17,844 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:34:17,844 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:34:17,854 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:34:18,216 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:34:18,889 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:34:20,198 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:34:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:34:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:51.226294+00:00 (in 29.999540 seconds) autopruneworker stdout | 2025-11-04 09:34:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:51 GMT)" (scheduled at 2025-11-04 09:34:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:34:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248801234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:34:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:34:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:34:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:34:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:34:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:32.435813+00:00 (in 11.000935 seconds) namespacegcworker stdout | 2025-11-04 09:34:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:21 GMT)" (scheduled at 2025-11-04 09:34:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:34:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:34:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 21, 435149), True, datetime.datetime(2025, 11, 4, 9, 34, 21, 435149), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:34:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:34:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:34:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:34:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:31.530064+00:00 (in 9.999589 seconds) notificationworker stdout | 2025-11-04 09:34:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:31 GMT)" (scheduled at 2025-11-04 09:34:21.530064+00:00) notificationworker stdout | 2025-11-04 09:34:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:34:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 21, 530764), True, datetime.datetime(2025, 11, 4, 9, 34, 21, 530764), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:34:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:34:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:34:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:34:22,477 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:34:25,325 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:34:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:37.122849+00:00 (in 9.999597 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:37 GMT)" (scheduled at 2025-11-04 09:34:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:34:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 27, 123568), True, datetime.datetime(2025, 11, 4, 9, 34, 27, 123568), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:34:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:34:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:34:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:34:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:34:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:57.222846+00:00 (in 29.999585 seconds) buildlogsarchiver stdout | 2025-11-04 09:34:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:57 GMT)" (scheduled at 2025-11-04 09:34:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:34:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 34, 27, 223481), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:34:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:34:27,239 [67] [DEBUG] [__main__] Archiving: 4708783e-59eb-4e42-b7a1-1923a4b4caff buildlogsarchiver stdout | 2025-11-04 09:34:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:34:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:34:27,247 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:34:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:34:27,558 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:34:27,855 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:34:27,868 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:34:27,868 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:34:27,879 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:34:28,269 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:34:28,921 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:34:29,304 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:34:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:34:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:44.351216+00:00 (in 14.999553 seconds) quotatotalworker stdout | 2025-11-04 09:34:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:44 GMT)" (scheduled at 2025-11-04 09:34:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:34:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:34:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:34:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:34:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:34:29,708 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:29,984 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:30,173 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:30,183 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:30,276 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:30,697 [286] [DEBUG] [app] Starting request: urn:request:51ae8cb8-6cc6-4ea3-9606-e4900f9715bd (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:30,697 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:30,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:30,701 [299] [DEBUG] [app] Starting request: urn:request:3eaa088f-64fa-45a4-8cd9-af1d4ad89b91 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:30,701 [299] [DEBUG] [app] Ending request: urn:request:3eaa088f-64fa-45a4-8cd9-af1d4ad89b91 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3eaa088f-64fa-45a4-8cd9-af1d4ad89b91', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:34:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:34:30,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:30,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:30,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:30,705 [285] [DEBUG] [app] Starting request: urn:request:ea9d5f0b-88ed-420e-b603-b035bec30828 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:30,706 [285] [DEBUG] [app] Ending request: urn:request:ea9d5f0b-88ed-420e-b603-b035bec30828 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ea9d5f0b-88ed-420e-b603-b035bec30828', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:34:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:30,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:30,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:30,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:30,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:30,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:30,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:30,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:30,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:30,730 [286] [DEBUG] [app] Ending request: urn:request:51ae8cb8-6cc6-4ea3-9606-e4900f9715bd (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:51ae8cb8-6cc6-4ea3-9606-e4900f9715bd', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:30,730 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:34:30,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:34:30,783 [285] [DEBUG] [app] Starting request: urn:request:952a8702-8398-47d9-958e-060cde1eb834 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:30,788 [292] [DEBUG] [app] Starting request: urn:request:a385539b-4284-4364-af41-d1db379c99cd (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:30,788 [292] [DEBUG] [app] Ending request: urn:request:a385539b-4284-4364-af41-d1db379c99cd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a385539b-4284-4364-af41-d1db379c99cd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:34:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:30,792 [287] [DEBUG] [app] Starting request: urn:request:6be235df-e731-4d5f-86e7-0678bcaf97a8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:30,793 [287] [DEBUG] [app] Ending request: urn:request:6be235df-e731-4d5f-86e7-0678bcaf97a8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6be235df-e731-4d5f-86e7-0678bcaf97a8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:34:30,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:34:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:30,801 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:30,801 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:30,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:30,816 [285] [DEBUG] [app] Ending request: urn:request:952a8702-8398-47d9-958e-060cde1eb834 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:952a8702-8398-47d9-958e-060cde1eb834', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:30,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:34:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) notificationworker stdout | 2025-11-04 09:34:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:40.533243+00:00 (in 9.002730 seconds) notificationworker stdout | 2025-11-04 09:34:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:41 GMT)" (scheduled at 2025-11-04 09:34:31.530064+00:00) notificationworker stdout | 2025-11-04 09:34:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:34:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 31, 530906), True, datetime.datetime(2025, 11, 4, 9, 34, 31, 530906), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:34:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:34:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:34:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:41 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:34:31,562 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:34:32,347 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:34:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:34:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:21.434460+00:00 (in 48.998309 seconds) namespacegcworker stdout | 2025-11-04 09:34:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:32 GMT)" (scheduled at 2025-11-04 09:34:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:34:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:34:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:32 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:34:32,543 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:34:32,557 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:34:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:34:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:48.625192+00:00 (in 15.001805 seconds) exportactionlogsworker stdout | 2025-11-04 09:34:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:33 GMT)" (scheduled at 2025-11-04 09:34:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:34:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:34:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 33, 623683), True, datetime.datetime(2025, 11, 4, 9, 34, 33, 623683), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:34:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:34:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:34:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:34:33,834 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:34:33,834 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:34:33,938 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:34:34,711 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:34:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:34:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:38.437104+00:00 (in 3.000873 seconds) gcworker stdout | 2025-11-04 09:34:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:05 GMT)" (scheduled at 2025-11-04 09:34:35.435809+00:00) gcworker stdout | 2025-11-04 09:34:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039275436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:34:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:34:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:34:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:34:36,024 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:34:36,393 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:34:37,020 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:34:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:40.124445+00:00 (in 3.001160 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:47 GMT)" (scheduled at 2025-11-04 09:34:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:34:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 37, 123598), True, datetime.datetime(2025, 11, 4, 9, 34, 37, 123598), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:34:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:34:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:34:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:34:37,449 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:34:37,879 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:34:37,893 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:34:37,893 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:34:37,903 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:34:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:34:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:05.435809+00:00 (in 26.998334 seconds) gcworker stdout | 2025-11-04 09:34:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:08 GMT)" (scheduled at 2025-11-04 09:34:38.437104+00:00) gcworker stdout | 2025-11-04 09:34:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:34:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248818449, None, 1, 0]) gcworker stdout | 2025-11-04 09:34:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:34:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:34:38,466 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,471 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,477 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,506 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,520 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,533 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,556 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,726 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:34:38,727 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:34:38,863 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:34:38,866 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:34:38,867 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:34:39,564 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:34:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:34:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:01.922827+00:00 (in 21.997738 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:34:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:39 GMT)" (scheduled at 2025-11-04 09:34:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:34:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:34:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:34:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:47.122849+00:00 (in 6.997898 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:40 GMT)" (scheduled at 2025-11-04 09:34:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:34:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:40 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:34:40,533 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:40,533 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:41.530064+00:00 (in 0.996351 seconds) notificationworker stdout | 2025-11-04 09:34:40,533 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:39:40 GMT)" (scheduled at 2025-11-04 09:34:40.533243+00:00) notificationworker stdout | 2025-11-04 09:34:40,534 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 34, 40, 534090), 'notification/%']) notificationworker stdout | 2025-11-04 09:34:40,546 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 34, 40, 534090), True, datetime.datetime(2025, 11, 4, 9, 34, 40, 534090), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:34:40,549 [88] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 34, 40, 534090), True, datetime.datetime(2025, 11, 4, 9, 34, 40, 534090), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 34, 40, 534090), 'notification/%']) notificationworker stdout | 2025-11-04 09:34:40,553 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:34:40,553 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:39:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:34:41,341 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:34:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:51.530064+00:00 (in 9.999613 seconds) notificationworker stdout | 2025-11-04 09:34:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:51 GMT)" (scheduled at 2025-11-04 09:34:41.530064+00:00) notificationworker stdout | 2025-11-04 09:34:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:34:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 41, 530738), True, datetime.datetime(2025, 11, 4, 9, 34, 41, 530738), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:34:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:34:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:34:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:34:42,789 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:34:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:34:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:57.523915+00:00 (in 14.090656 seconds) repositorygcworker stdout | 2025-11-04 09:34:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:43 GMT)" (scheduled at 2025-11-04 09:34:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:34:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:34:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 43, 433505), True, datetime.datetime(2025, 11, 4, 9, 34, 43, 433505), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:34:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:34:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:34:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:34:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:34:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:59.351216+00:00 (in 14.999575 seconds) quotatotalworker stdout | 2025-11-04 09:34:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:59 GMT)" (scheduled at 2025-11-04 09:34:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:34:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:34:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:34:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:34:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:34:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:34:45,289 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:34:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:34:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:47.479080+00:00 (in 1.998532 seconds) securityworker stdout | 2025-11-04 09:34:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:15 GMT)" (scheduled at 2025-11-04 09:34:45.479769+00:00) securityworker stdout | 2025-11-04 09:34:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:34:45,481 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:34:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:34:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:45,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 29, 45, 484322), 1, 3]) securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 29, 45, 484322), 1, 3]) securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:34:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:34:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:34:45,697 [288] [DEBUG] [app] Starting request: urn:request:d1522b6c-3dc1-4180-acd9-5c7076d199c3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:45,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:45,701 [299] [DEBUG] [app] Starting request: urn:request:64543963-1da7-42e2-86d4-815412448a1c (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:45,701 [299] [DEBUG] [app] Ending request: urn:request:64543963-1da7-42e2-86d4-815412448a1c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:64543963-1da7-42e2-86d4-815412448a1c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:34:45,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:45,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:45,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:45,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:45,705 [285] [DEBUG] [app] Starting request: urn:request:d9637260-4035-4941-a85d-43e663970c96 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:45,706 [285] [DEBUG] [app] Ending request: urn:request:d9637260-4035-4941-a85d-43e663970c96 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d9637260-4035-4941-a85d-43e663970c96', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:34:45,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:34:45,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:45,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:45,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:45,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:45,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:45,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:45,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:45,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:45,730 [288] [DEBUG] [app] Ending request: urn:request:d1522b6c-3dc1-4180-acd9-5c7076d199c3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d1522b6c-3dc1-4180-acd9-5c7076d199c3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:45,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:34:45,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:34:45,783 [285] [DEBUG] [app] Starting request: urn:request:67b66cc4-7024-4851-8ada-b86dbee45d52 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:34:45,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:45,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:34:45,788 [299] [DEBUG] [app] Starting request: urn:request:c3872b6e-d6c3-4c05-b769-56571ee86234 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:34:45,788 [299] [DEBUG] [app] Ending request: urn:request:c3872b6e-d6c3-4c05-b769-56571ee86234 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c3872b6e-d6c3-4c05-b769-56571ee86234', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:34:45,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:34:45,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:45,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:34:45,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:34:45,792 [287] [DEBUG] [app] Starting request: urn:request:d783d17c-6728-4090-b65d-2d8a05aae2ba (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:34:45,792 [287] [DEBUG] [app] Ending request: urn:request:d783d17c-6728-4090-b65d-2d8a05aae2ba (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d783d17c-6728-4090-b65d-2d8a05aae2ba', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:34:45,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:34:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:34:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:34:45,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:34:45,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:34:45,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:34:45,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:34:45,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:34:45,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:34:45,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:34:45,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:34:45,816 [285] [DEBUG] [app] Ending request: urn:request:67b66cc4-7024-4851-8ada-b86dbee45d52 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:67b66cc4-7024-4851-8ada-b86dbee45d52', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:34:45,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:34:45,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:34:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:34:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:34:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:53.124163+00:00 (in 6.000893 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:57 GMT)" (scheduled at 2025-11-04 09:34:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:34:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 47, 123559), True, datetime.datetime(2025, 11, 4, 9, 34, 47, 123559), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:34:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:34:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:34:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:34:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:34:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:34:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:15.479769+00:00 (in 28.000209 seconds) securityworker stdout | 2025-11-04 09:34:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:17 GMT)" (scheduled at 2025-11-04 09:34:47.479080+00:00) securityworker stdout | 2025-11-04 09:34:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:34:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:34:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:34:47,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:34:47,496 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:47,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:47,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:47,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:47,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:47,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:47,504 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:47,505 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 29, 47, 484301), 1, 3]) securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:34:47,508 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:34:47,509 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:34:47,509 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:34:47,509 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 29, 47, 484301), 1, 3]) securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:34:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:34:47,513 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:34:47,513 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:17 GMT)" executed successfully builder stdout | 2025-11-04 09:34:47,904 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:34:47,917 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:34:47,917 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:34:47,927 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:34:48,225 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:34:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:34:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:33.622985+00:00 (in 44.997382 seconds) exportactionlogsworker stdout | 2025-11-04 09:34:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:48 GMT)" (scheduled at 2025-11-04 09:34:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:34:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:34:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:34:48,897 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:34:50,207 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:34:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:34:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:21.226294+00:00 (in 29.999552 seconds) autopruneworker stdout | 2025-11-04 09:34:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:21 GMT)" (scheduled at 2025-11-04 09:34:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:34:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248831234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:34:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:34:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:34:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:34:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:34:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:01.530064+00:00 (in 9.999557 seconds) notificationworker stdout | 2025-11-04 09:34:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:01 GMT)" (scheduled at 2025-11-04 09:34:51.530064+00:00) notificationworker stdout | 2025-11-04 09:34:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:34:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 51, 530780), True, datetime.datetime(2025, 11, 4, 9, 34, 51, 530780), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:34:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:34:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:34:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:34:52,486 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:34:53,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:53,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:34:57.122849+00:00 (in 3.998264 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:53,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:39:53 GMT)" (scheduled at 2025-11-04 09:34:53.124163+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:53,125 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 34, 53, 124825), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:34:53,138 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 34, 53, 124825), True, datetime.datetime(2025, 11, 4, 9, 34, 53, 124825), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:34:53,141 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 34, 53, 124825), True, datetime.datetime(2025, 11, 4, 9, 34, 53, 124825), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 34, 53, 124825), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:34:53,145 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:34:53,145 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:39:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:34:55,333 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:34:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:34:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:07.122849+00:00 (in 9.999608 seconds) proxycacheblobworker stdout | 2025-11-04 09:34:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:07 GMT)" (scheduled at 2025-11-04 09:34:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:34:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:34:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 34, 57, 123527), True, datetime.datetime(2025, 11, 4, 9, 34, 57, 123527), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:34:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:34:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:09.133645+00:00 (in 12.001482 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:34:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:57 GMT)" (scheduled at 2025-11-04 09:34:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:34:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:34:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:34:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:34:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:34:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:34:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:34:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:34:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:34:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:27.222846+00:00 (in 29.999601 seconds) buildlogsarchiver stdout | 2025-11-04 09:34:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:27 GMT)" (scheduled at 2025-11-04 09:34:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:34:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 34, 57, 223422), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:34:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:34:57,238 [67] [DEBUG] [__main__] Archiving: 4708783e-59eb-4e42-b7a1-1923a4b4caff buildlogsarchiver stdout | 2025-11-04 09:34:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:34:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:34:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:34:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:34:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:43.432867+00:00 (in 45.908521 seconds) repositorygcworker stdout | 2025-11-04 09:34:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:57 GMT)" (scheduled at 2025-11-04 09:34:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:34:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:34:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:35:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:34:57,567 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:34:57,928 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:34:57,941 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:34:57,941 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:34:57,951 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:34:58,278 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:34:58,930 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:34:59,313 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:34:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:34:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:14.351216+00:00 (in 14.999538 seconds) quotatotalworker stdout | 2025-11-04 09:34:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:14 GMT)" (scheduled at 2025-11-04 09:34:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:34:59,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:34:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:34:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:34:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:34:59,732 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:34:59,996 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:00,187 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:00,197 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:00,290 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:00,697 [288] [DEBUG] [app] Starting request: urn:request:f19ab1ee-d167-4346-9662-8cb8d33f3f16 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:00,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:00,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:00,701 [298] [DEBUG] [app] Starting request: urn:request:71235b9f-ce67-48b8-91a5-2fe70702d409 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:00,701 [298] [DEBUG] [app] Ending request: urn:request:71235b9f-ce67-48b8-91a5-2fe70702d409 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:71235b9f-ce67-48b8-91a5-2fe70702d409', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:35:00,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:35:00,702 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:00,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:00,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:00,705 [287] [DEBUG] [app] Starting request: urn:request:766beb59-5b04-4db1-a480-5d5942c062df (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:00,706 [287] [DEBUG] [app] Ending request: urn:request:766beb59-5b04-4db1-a480-5d5942c062df (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:766beb59-5b04-4db1-a480-5d5942c062df', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:35:00,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:35:00,706 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:00,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:00,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:00,714 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:00,714 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:00,724 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:00,727 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:00,730 [288] [DEBUG] [app] Ending request: urn:request:f19ab1ee-d167-4346-9662-8cb8d33f3f16 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f19ab1ee-d167-4346-9662-8cb8d33f3f16', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:00,730 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:35:00,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:35:00,784 [287] [DEBUG] [app] Starting request: urn:request:f9994baf-b191-4845-ad4c-fb457eef1e42 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:00,784 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:00,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:00,788 [296] [DEBUG] [app] Starting request: urn:request:c7248544-b187-4898-95fe-d2c84349153a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:00,788 [296] [DEBUG] [app] Ending request: urn:request:c7248544-b187-4898-95fe-d2c84349153a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c7248544-b187-4898-95fe-d2c84349153a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:35:00,788 [296] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:35:00,788 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:00,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:00,791 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:00,792 [285] [DEBUG] [app] Starting request: urn:request:4a94a415-38a7-4f41-9d43-1223ca3f6b2d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:00,792 [285] [DEBUG] [app] Ending request: urn:request:4a94a415-38a7-4f41-9d43-1223ca3f6b2d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4a94a415-38a7-4f41-9d43-1223ca3f6b2d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:35:00,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:35:00,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:00,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:00,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:00,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:00,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:00,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:00,812 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:00,816 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:00,819 [287] [DEBUG] [app] Ending request: urn:request:f9994baf-b191-4845-ad4c-fb457eef1e42 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f9994baf-b191-4845-ad4c-fb457eef1e42', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:00,819 [287] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.036 47 0.037) gunicorn-web stdout | 2025-11-04 09:35:00,819 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" notificationworker stdout | 2025-11-04 09:35:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:35:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:11.530064+00:00 (in 9.999589 seconds) notificationworker stdout | 2025-11-04 09:35:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:11 GMT)" (scheduled at 2025-11-04 09:35:01.530064+00:00) notificationworker stdout | 2025-11-04 09:35:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:35:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 1, 530750), True, datetime.datetime(2025, 11, 4, 9, 35, 1, 530750), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:35:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:35:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:35:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:11 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:35:01,570 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:35:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:35:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:39.924652+00:00 (in 38.001432 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:35:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:01 GMT)" (scheduled at 2025-11-04 09:35:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:35:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:35:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 1, 923937), True, datetime.datetime(2025, 11, 4, 9, 35, 1, 923937), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:35:01,936 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:35:01,936 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:35:01,936 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:35:02,366 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:35:02,552 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:35:02,566 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:35:03,843 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:35:03,843 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:35:03,946 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:35:04,720 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:35:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:35:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:08.437104+00:00 (in 3.000916 seconds) gcworker stdout | 2025-11-04 09:35:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:35 GMT)" (scheduled at 2025-11-04 09:35:05.435809+00:00) gcworker stdout | 2025-11-04 09:35:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039305436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:35:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:35:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:35:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:35:06,033 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:35:06,402 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:35:07,029 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:35:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:35:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:17.122849+00:00 (in 9.999567 seconds) proxycacheblobworker stdout | 2025-11-04 09:35:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:17 GMT)" (scheduled at 2025-11-04 09:35:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:35:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:35:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 7, 123622), True, datetime.datetime(2025, 11, 4, 9, 35, 7, 123622), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:35:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:35:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:35:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:35:07,458 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:35:07,951 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:35:07,965 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:35:07,965 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:35:07,975 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:35:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:35:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:35.435809+00:00 (in 26.998340 seconds) gcworker stdout | 2025-11-04 09:35:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:38 GMT)" (scheduled at 2025-11-04 09:35:08.437104+00:00) gcworker stdout | 2025-11-04 09:35:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:35:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248848449, None, 1, 0]) gcworker stdout | 2025-11-04 09:35:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:35:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:35:08,477 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,482 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,487 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,520 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,532 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,547 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,569 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,740 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:08,741 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:35:08,874 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:35:08,881 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:35:08,881 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:35:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:35:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:57.131795+00:00 (in 47.997700 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:35:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:09 GMT)" (scheduled at 2025-11-04 09:35:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:35:09,135 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:35:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:35:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:35:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:35:09,584 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:35:11,350 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:35:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:35:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:15.535095+00:00 (in 4.004584 seconds) notificationworker stdout | 2025-11-04 09:35:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:21 GMT)" (scheduled at 2025-11-04 09:35:11.530064+00:00) notificationworker stdout | 2025-11-04 09:35:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:35:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 11, 530788), True, datetime.datetime(2025, 11, 4, 9, 35, 11, 530788), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:35:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:35:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:35:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:35:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:35:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:12.222828+00:00 (in 59.999581 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:35:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:12 GMT)" (scheduled at 2025-11-04 09:35:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:35:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:35:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:35:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:35:12,798 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:35:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:35:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:29.351216+00:00 (in 14.999620 seconds) quotatotalworker stdout | 2025-11-04 09:35:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:29 GMT)" (scheduled at 2025-11-04 09:35:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:35:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:35:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:35:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:35:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:35:15,298 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:35:15,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:35:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:17.479080+00:00 (in 1.998885 seconds) securityworker stdout | 2025-11-04 09:35:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:45 GMT)" (scheduled at 2025-11-04 09:35:15.479769+00:00) securityworker stdout | 2025-11-04 09:35:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:35:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:35:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:35:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:35:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:15,496 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:15,500 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 30, 15, 484240), 1, 3]) securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:15,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 30, 15, 484240), 1, 3]) securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:35:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:35:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:35:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:35:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:21.530064+00:00 (in 5.994518 seconds) notificationworker stdout | 2025-11-04 09:35:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:15 GMT)" (scheduled at 2025-11-04 09:35:15.535095+00:00) notificationworker stdout | 2025-11-04 09:35:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:35:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:35:15,697 [287] [DEBUG] [app] Starting request: urn:request:a4d9a0da-d5c6-4ba6-9e73-b599358b9d99 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:15,698 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:15,700 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:15,701 [298] [DEBUG] [app] Starting request: urn:request:8dc5a083-4e95-4501-ac9b-8d6f87182e21 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:15,701 [298] [DEBUG] [app] Ending request: urn:request:8dc5a083-4e95-4501-ac9b-8d6f87182e21 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8dc5a083-4e95-4501-ac9b-8d6f87182e21', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:35:15,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:35:15,702 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:15,703 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:15,705 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:15,705 [285] [DEBUG] [app] Starting request: urn:request:ba2cd103-efbf-410e-aad9-cd1391b5659a (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:15,706 [285] [DEBUG] [app] Ending request: urn:request:ba2cd103-efbf-410e-aad9-cd1391b5659a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ba2cd103-efbf-410e-aad9-cd1391b5659a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:35:15,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:35:15,706 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:15,707 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:15,707 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:15,707 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:15,714 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:15,714 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:15,725 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:15,728 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:15,731 [287] [DEBUG] [app] Ending request: urn:request:a4d9a0da-d5c6-4ba6-9e73-b599358b9d99 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a4d9a0da-d5c6-4ba6-9e73-b599358b9d99', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:15,731 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:35:15,731 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.036) gunicorn-web stdout | 2025-11-04 09:35:15,783 [285] [DEBUG] [app] Starting request: urn:request:a9745971-4c88-4205-a2aa-19aca722c3f0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:15,786 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:15,787 [299] [DEBUG] [app] Starting request: urn:request:362ecd73-533e-4d5b-bdb9-8526352ae241 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:15,788 [299] [DEBUG] [app] Ending request: urn:request:362ecd73-533e-4d5b-bdb9-8526352ae241 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:362ecd73-533e-4d5b-bdb9-8526352ae241', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:35:15,788 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:35:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:15,791 [287] [DEBUG] [app] Starting request: urn:request:0509be2f-7e65-4729-b10c-f1347ab86c81 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:15,792 [287] [DEBUG] [app] Ending request: urn:request:0509be2f-7e65-4729-b10c-f1347ab86c81 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0509be2f-7e65-4729-b10c-f1347ab86c81', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:35:15,792 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:35:15,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:15,792 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:15,792 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:15,792 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:15,799 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:15,799 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:15,809 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:15,812 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:15,815 [285] [DEBUG] [app] Ending request: urn:request:a9745971-4c88-4205-a2aa-19aca722c3f0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a9745971-4c88-4205-a2aa-19aca722c3f0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:15,815 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:35:15,815 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.033) proxycacheblobworker stdout | 2025-11-04 09:35:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:35:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:27.122849+00:00 (in 9.999582 seconds) proxycacheblobworker stdout | 2025-11-04 09:35:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:27 GMT)" (scheduled at 2025-11-04 09:35:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:35:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:35:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 17, 123565), True, datetime.datetime(2025, 11, 4, 9, 35, 17, 123565), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:35:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:35:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:35:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:35:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:35:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:45.479769+00:00 (in 28.000238 seconds) securityworker stdout | 2025-11-04 09:35:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:47 GMT)" (scheduled at 2025-11-04 09:35:17.479080+00:00) securityworker stdout | 2025-11-04 09:35:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:35:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:35:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:35:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:35:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:35:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:47 GMT)" executed successfully builder stdout | 2025-11-04 09:35:17,975 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:35:17,989 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:35:17,989 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:35:17,999 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:35:18,234 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:35:18,906 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:35:20,216 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:35:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:35:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:51.226294+00:00 (in 29.999557 seconds) autopruneworker stdout | 2025-11-04 09:35:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:51 GMT)" (scheduled at 2025-11-04 09:35:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:35:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248861235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:35:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:35:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:35:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:35:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:35:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:32.435813+00:00 (in 11.000981 seconds) namespacegcworker stdout | 2025-11-04 09:35:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:21 GMT)" (scheduled at 2025-11-04 09:35:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:35:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:35:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 21, 435125), True, datetime.datetime(2025, 11, 4, 9, 35, 21, 435125), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:35:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:35:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:35:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:35:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:35:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:31.530064+00:00 (in 9.999562 seconds) notificationworker stdout | 2025-11-04 09:35:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:31 GMT)" (scheduled at 2025-11-04 09:35:21.530064+00:00) notificationworker stdout | 2025-11-04 09:35:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:35:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 21, 530765), True, datetime.datetime(2025, 11, 4, 9, 35, 21, 530765), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:35:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:35:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:35:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:35:22,495 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:35:25,342 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:35:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:35:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:37.122849+00:00 (in 9.999578 seconds) proxycacheblobworker stdout | 2025-11-04 09:35:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:37 GMT)" (scheduled at 2025-11-04 09:35:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:35:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:35:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 27, 123612), True, datetime.datetime(2025, 11, 4, 9, 35, 27, 123612), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:35:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:35:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:35:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:35:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:35:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:57.222846+00:00 (in 29.999593 seconds) buildlogsarchiver stdout | 2025-11-04 09:35:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:57 GMT)" (scheduled at 2025-11-04 09:35:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:35:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 35, 27, 223480), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:35:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:35:27,238 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:35:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:35:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:35:27,245 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:35:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:35:27,575 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:35:27,999 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:35:28,013 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:35:28,013 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:35:28,023 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:35:28,287 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:35:28,938 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:35:29,322 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:35:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:35:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:44.351216+00:00 (in 14.999569 seconds) quotatotalworker stdout | 2025-11-04 09:35:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:44 GMT)" (scheduled at 2025-11-04 09:35:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:35:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:35:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:35:29,365 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:35:29,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:35:29,743 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:30,012 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:30,201 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:30,211 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:30,304 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:35:30,697 [288] [DEBUG] [app] Starting request: urn:request:828a9fbc-fd89-4e59-aca0-81f9f269fef5 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:30,698 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:30,701 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:30,702 [297] [DEBUG] [app] Starting request: urn:request:c2ad6a90-e949-474b-b96b-f53e11518b87 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:30,702 [297] [DEBUG] [app] Ending request: urn:request:c2ad6a90-e949-474b-b96b-f53e11518b87 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c2ad6a90-e949-474b-b96b-f53e11518b87', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:35:30,702 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:35:30,703 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:30,703 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:30,705 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:30,706 [285] [DEBUG] [app] Starting request: urn:request:6b63031b-5699-418e-89ca-4bfbdd300b4c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:30,706 [285] [DEBUG] [app] Ending request: urn:request:6b63031b-5699-418e-89ca-4bfbdd300b4c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6b63031b-5699-418e-89ca-4bfbdd300b4c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:35:30,707 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:35:30,707 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:30,707 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:30,707 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:30,707 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:30,715 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:30,715 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:30,725 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:30,728 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:30,731 [288] [DEBUG] [app] Ending request: urn:request:828a9fbc-fd89-4e59-aca0-81f9f269fef5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:828a9fbc-fd89-4e59-aca0-81f9f269fef5', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:30,731 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:35:30,731 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:35:30,784 [285] [DEBUG] [app] Starting request: urn:request:00eb5e20-02de-45ce-9620-e601d8a50716 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:30,785 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:30,788 [292] [DEBUG] [app] Starting request: urn:request:56787332-2b7d-4701-9ffd-8b2b46d9fb4b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:30,788 [292] [DEBUG] [app] Ending request: urn:request:56787332-2b7d-4701-9ffd-8b2b46d9fb4b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:56787332-2b7d-4701-9ffd-8b2b46d9fb4b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:35:30,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:35:30,789 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:30,792 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:30,792 [288] [DEBUG] [app] Starting request: urn:request:0b06f7b4-cde4-46d0-9b16-4386d9510f40 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:30,792 [288] [DEBUG] [app] Ending request: urn:request:0b06f7b4-cde4-46d0-9b16-4386d9510f40 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0b06f7b4-cde4-46d0-9b16-4386d9510f40', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:35:30,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:35:30,793 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:30,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:30,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:30,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:30,817 [285] [DEBUG] [app] Ending request: urn:request:00eb5e20-02de-45ce-9620-e601d8a50716 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:00eb5e20-02de-45ce-9620-e601d8a50716', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:30,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:35:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) notificationworker stdout | 2025-11-04 09:35:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:35:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:41.530064+00:00 (in 9.999600 seconds) notificationworker stdout | 2025-11-04 09:35:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:41 GMT)" (scheduled at 2025-11-04 09:35:31.530064+00:00) notificationworker stdout | 2025-11-04 09:35:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:35:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 31, 530755), True, datetime.datetime(2025, 11, 4, 9, 35, 31, 530755), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:35:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:35:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:35:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:41 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:35:31,579 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:35:32,375 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:35:32,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:35:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:21.434460+00:00 (in 48.998291 seconds) namespacegcworker stdout | 2025-11-04 09:35:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:32 GMT)" (scheduled at 2025-11-04 09:35:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:35:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:35:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:32 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:35:32,561 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:35:32,575 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:35:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:35:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:48.625192+00:00 (in 15.001801 seconds) exportactionlogsworker stdout | 2025-11-04 09:35:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:33 GMT)" (scheduled at 2025-11-04 09:35:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:35:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:35:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 33, 623686), True, datetime.datetime(2025, 11, 4, 9, 35, 33, 623686), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:35:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:35:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:35:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:35:33,851 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:35:33,851 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:35:33,955 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:35:34,729 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:35:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:35:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:38.437104+00:00 (in 3.000909 seconds) gcworker stdout | 2025-11-04 09:35:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:05 GMT)" (scheduled at 2025-11-04 09:35:35.435809+00:00) gcworker stdout | 2025-11-04 09:35:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039335436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:35:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:35:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:35:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:35:36,041 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:35:36,411 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:35:37,038 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:35:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:35:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:40.124445+00:00 (in 3.001155 seconds) proxycacheblobworker stdout | 2025-11-04 09:35:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:47 GMT)" (scheduled at 2025-11-04 09:35:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:35:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:35:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 37, 123590), True, datetime.datetime(2025, 11, 4, 9, 35, 37, 123590), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:35:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:35:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:35:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:35:37,467 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:35:38,023 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:35:38,036 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:35:38,036 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:35:38,046 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:35:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:35:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:05.435809+00:00 (in 26.998335 seconds) gcworker stdout | 2025-11-04 09:35:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:08 GMT)" (scheduled at 2025-11-04 09:35:38.437104+00:00) gcworker stdout | 2025-11-04 09:35:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:35:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248878449, None, 1, 0]) gcworker stdout | 2025-11-04 09:35:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:35:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:35:38,488 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,493 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,497 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,533 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,543 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,559 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,583 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,753 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:35:38,754 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:35:38,884 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:35:38,896 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:35:38,896 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:35:39,593 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:35:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:35:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:01.922827+00:00 (in 21.997778 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:35:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:39 GMT)" (scheduled at 2025-11-04 09:35:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:35:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:35:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:35:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:35:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:47.122849+00:00 (in 6.997978 seconds) proxycacheblobworker stdout | 2025-11-04 09:35:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:40 GMT)" (scheduled at 2025-11-04 09:35:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:35:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:35:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:35:41,359 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:35:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:35:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:51.530064+00:00 (in 9.999558 seconds) notificationworker stdout | 2025-11-04 09:35:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:51 GMT)" (scheduled at 2025-11-04 09:35:41.530064+00:00) notificationworker stdout | 2025-11-04 09:35:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:35:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 41, 530765), True, datetime.datetime(2025, 11, 4, 9, 35, 41, 530765), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:35:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:35:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:35:41,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:35:42,806 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:35:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:35:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:57.523915+00:00 (in 14.090600 seconds) repositorygcworker stdout | 2025-11-04 09:35:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:43 GMT)" (scheduled at 2025-11-04 09:35:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:35:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:35:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 43, 433599), True, datetime.datetime(2025, 11, 4, 9, 35, 43, 433599), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:35:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:35:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:35:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:35:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:35:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:59.351216+00:00 (in 14.999612 seconds) quotatotalworker stdout | 2025-11-04 09:35:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:59 GMT)" (scheduled at 2025-11-04 09:35:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:35:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:35:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:35:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:35:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:35:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:35:45,307 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:35:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:35:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:47.479080+00:00 (in 1.998904 seconds) securityworker stdout | 2025-11-04 09:35:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:15 GMT)" (scheduled at 2025-11-04 09:35:45.479769+00:00) securityworker stdout | 2025-11-04 09:35:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:35:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:35:45,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:35:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:35:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:45,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:45,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:45,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:45,498 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 30, 45, 483828), 1, 3]) securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 30, 45, 483828), 1, 3]) securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:35:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:35:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:35:45,696 [286] [DEBUG] [app] Starting request: urn:request:18f685b9-6e41-4e8b-9b18-a4e89bdc642f (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:45,697 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:45,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:45,701 [292] [DEBUG] [app] Starting request: urn:request:ebc0a2f4-d7ae-466a-bb62-cada8a71828d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:45,701 [292] [DEBUG] [app] Ending request: urn:request:ebc0a2f4-d7ae-466a-bb62-cada8a71828d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ebc0a2f4-d7ae-466a-bb62-cada8a71828d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:35:45,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:35:45,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:45,702 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:45,704 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:45,705 [288] [DEBUG] [app] Starting request: urn:request:81b40778-183c-4d78-8f0f-694326ea871d (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:45,706 [288] [DEBUG] [app] Ending request: urn:request:81b40778-183c-4d78-8f0f-694326ea871d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:81b40778-183c-4d78-8f0f-694326ea871d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:35:45,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:35:45,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:45,706 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:45,706 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:45,706 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:45,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:45,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:45,723 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:45,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:45,729 [286] [DEBUG] [app] Ending request: urn:request:18f685b9-6e41-4e8b-9b18-a4e89bdc642f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:18f685b9-6e41-4e8b-9b18-a4e89bdc642f', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:45,730 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:35:45,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:35:45,783 [288] [DEBUG] [app] Starting request: urn:request:854cd654-ad05-41f4-9187-82a8e40e6ed6 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:35:45,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:45,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:35:45,788 [292] [DEBUG] [app] Starting request: urn:request:cc0ffa60-5bab-4eb1-92dc-1f780c22aacd (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:35:45,788 [292] [DEBUG] [app] Ending request: urn:request:cc0ffa60-5bab-4eb1-92dc-1f780c22aacd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cc0ffa60-5bab-4eb1-92dc-1f780c22aacd', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:35:45,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:35:45,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:45,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:35:45,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:35:45,792 [285] [DEBUG] [app] Starting request: urn:request:8e0187ac-bf93-478f-b9dd-69464ae4f4df (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:35:45,792 [285] [DEBUG] [app] Ending request: urn:request:8e0187ac-bf93-478f-b9dd-69464ae4f4df (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8e0187ac-bf93-478f-b9dd-69464ae4f4df', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:35:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:35:45,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:35:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:35:45,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:35:45,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:35:45,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:35:45,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:35:45,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:35:45,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:35:45,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:35:45,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:35:45,816 [288] [DEBUG] [app] Ending request: urn:request:854cd654-ad05-41f4-9187-82a8e40e6ed6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:854cd654-ad05-41f4-9187-82a8e40e6ed6', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:35:45,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:35:45,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:35:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:35:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:35:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:35:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:35:57.122849+00:00 (in 9.999592 seconds) proxycacheblobworker stdout | 2025-11-04 09:35:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:57 GMT)" (scheduled at 2025-11-04 09:35:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:35:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:35:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 47, 123545), True, datetime.datetime(2025, 11, 4, 9, 35, 47, 123545), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:35:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:35:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:35:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:35:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:35:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:35:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:15.479769+00:00 (in 28.000254 seconds) securityworker stdout | 2025-11-04 09:35:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:17 GMT)" (scheduled at 2025-11-04 09:35:47.479080+00:00) securityworker stdout | 2025-11-04 09:35:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:35:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:35:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:35:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:35:47,495 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:47,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:47,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:47,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:47,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:47,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:47,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:47,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:47,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:35:47,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 30, 47, 483038), 1, 3]) securityworker stdout | 2025-11-04 09:35:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:35:47,507 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 30, 47, 483038), 1, 3]) securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:35:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:35:47,511 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:35:47,511 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:17 GMT)" executed successfully builder stdout | 2025-11-04 09:35:48,047 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:35:48,060 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:35:48,061 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:35:48,071 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:35:48,243 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:35:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:35:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:33.622985+00:00 (in 44.997387 seconds) exportactionlogsworker stdout | 2025-11-04 09:35:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:48 GMT)" (scheduled at 2025-11-04 09:35:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:35:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:35:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:35:48,915 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:35:50,228 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:35:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:35:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:21.226294+00:00 (in 29.999584 seconds) autopruneworker stdout | 2025-11-04 09:35:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:21 GMT)" (scheduled at 2025-11-04 09:35:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:35:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248891234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:35:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:35:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:35:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:35:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:35:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:01.530064+00:00 (in 9.999621 seconds) notificationworker stdout | 2025-11-04 09:35:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:01 GMT)" (scheduled at 2025-11-04 09:35:51.530064+00:00) notificationworker stdout | 2025-11-04 09:35:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:35:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 51, 530696), True, datetime.datetime(2025, 11, 4, 9, 35, 51, 530696), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:35:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:35:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:35:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:35:52,504 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:35:55,352 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:35:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:35:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:07.122849+00:00 (in 9.999627 seconds) proxycacheblobworker stdout | 2025-11-04 09:35:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:07 GMT)" (scheduled at 2025-11-04 09:35:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:35:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:35:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 35, 57, 123513), True, datetime.datetime(2025, 11, 4, 9, 35, 57, 123513), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:35:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:35:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:09.133645+00:00 (in 12.001452 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:35:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:57 GMT)" (scheduled at 2025-11-04 09:35:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:35:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:35:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:35:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:35:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:35:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:35:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:35:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:35:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:35:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:27.222846+00:00 (in 29.999585 seconds) buildlogsarchiver stdout | 2025-11-04 09:35:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:27 GMT)" (scheduled at 2025-11-04 09:35:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:35:57,224 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 35, 57, 223837), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:35:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:35:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:35:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:35:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:35:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:35:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:35:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:43.432867+00:00 (in 45.908495 seconds) repositorygcworker stdout | 2025-11-04 09:35:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:57 GMT)" (scheduled at 2025-11-04 09:35:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:35:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:35:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:36:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:35:57,584 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:35:58,071 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:35:58,084 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:35:58,084 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:35:58,094 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:35:58,295 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:35:58,947 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:35:59,330 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:35:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:35:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:14.351216+00:00 (in 14.999639 seconds) quotatotalworker stdout | 2025-11-04 09:35:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:14 GMT)" (scheduled at 2025-11-04 09:35:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:35:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:35:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:35:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:35:59,365 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:35:59,753 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:00,024 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:00,215 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:00,225 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:00,318 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:00,697 [285] [DEBUG] [app] Starting request: urn:request:5df1f143-3537-42b8-bc3a-071edf8250b3 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:00,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:00,702 [298] [DEBUG] [app] Starting request: urn:request:740d5b6f-b965-418e-a5a8-ea5f7b5ae7af (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:00,702 [298] [DEBUG] [app] Ending request: urn:request:740d5b6f-b965-418e-a5a8-ea5f7b5ae7af (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:740d5b6f-b965-418e-a5a8-ea5f7b5ae7af', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:36:00,702 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:36:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:00,706 [287] [DEBUG] [app] Starting request: urn:request:c057c23f-e9c8-42a7-8330-dab3e1c502a5 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:00,706 [287] [DEBUG] [app] Ending request: urn:request:c057c23f-e9c8-42a7-8330-dab3e1c502a5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c057c23f-e9c8-42a7-8330-dab3e1c502a5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:36:00,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:36:00,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:00,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:00,731 [285] [DEBUG] [app] Ending request: urn:request:5df1f143-3537-42b8-bc3a-071edf8250b3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5df1f143-3537-42b8-bc3a-071edf8250b3', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:36:00,784 [286] [DEBUG] [app] Starting request: urn:request:b3010e65-d468-4a73-8a14-7bee18a6465c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:00,784 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:00,787 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:00,787 [298] [DEBUG] [app] Starting request: urn:request:61e07969-327d-4fd2-a10e-40280c0a72a3 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:00,788 [298] [DEBUG] [app] Ending request: urn:request:61e07969-327d-4fd2-a10e-40280c0a72a3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:61e07969-327d-4fd2-a10e-40280c0a72a3', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:36:00,788 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:00,788 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:00,789 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:00,791 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:00,792 [285] [DEBUG] [app] Starting request: urn:request:1479126d-4933-44cd-8b13-7f076e995a49 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:00,792 [285] [DEBUG] [app] Ending request: urn:request:1479126d-4933-44cd-8b13-7f076e995a49 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1479126d-4933-44cd-8b13-7f076e995a49', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:36:00,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:00,792 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:00,792 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:00,793 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:00,793 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:00,800 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:00,800 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:00,809 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:00,813 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:00,816 [286] [DEBUG] [app] Ending request: urn:request:b3010e65-d468-4a73-8a14-7bee18a6465c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b3010e65-d468-4a73-8a14-7bee18a6465c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:00,816 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:00,816 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.033) notificationworker stdout | 2025-11-04 09:36:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:36:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:11.530064+00:00 (in 9.999584 seconds) notificationworker stdout | 2025-11-04 09:36:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:11 GMT)" (scheduled at 2025-11-04 09:36:01.530064+00:00) notificationworker stdout | 2025-11-04 09:36:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:36:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 1, 530872), True, datetime.datetime(2025, 11, 4, 9, 36, 1, 530872), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:36:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:36:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:36:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:11 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:36:01,588 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:36:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:36:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:39.924652+00:00 (in 38.001449 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:36:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:01 GMT)" (scheduled at 2025-11-04 09:36:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:36:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:36:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 1, 923439), True, datetime.datetime(2025, 11, 4, 9, 36, 1, 923439), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:36:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:36:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:36:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:36:02,384 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:36:02,569 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:36:02,584 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:36:03,860 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:36:03,860 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:36:03,963 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:36:04,739 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:36:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:36:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:08.437104+00:00 (in 3.000905 seconds) gcworker stdout | 2025-11-04 09:36:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:35 GMT)" (scheduled at 2025-11-04 09:36:05.435809+00:00) gcworker stdout | 2025-11-04 09:36:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039365436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:36:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:36:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:36:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:36:06,050 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:36:06,420 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:36:07,047 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:36:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:36:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:17.122849+00:00 (in 9.999621 seconds) proxycacheblobworker stdout | 2025-11-04 09:36:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:17 GMT)" (scheduled at 2025-11-04 09:36:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:36:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:36:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 7, 123477), True, datetime.datetime(2025, 11, 4, 9, 36, 7, 123477), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:36:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:36:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:36:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:36:07,476 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:36:08,095 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:36:08,108 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:36:08,108 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:36:08,118 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:36:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:36:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:35.435809+00:00 (in 26.998335 seconds) gcworker stdout | 2025-11-04 09:36:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:38 GMT)" (scheduled at 2025-11-04 09:36:08.437104+00:00) gcworker stdout | 2025-11-04 09:36:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:36:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248908449, None, 1, 0]) gcworker stdout | 2025-11-04 09:36:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:36:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:36:08,499 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,504 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,512 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,546 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,554 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,571 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,596 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,765 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:08,771 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:36:08,894 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:36:08,906 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:36:08,906 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:36:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:36:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:57.131795+00:00 (in 47.997717 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:36:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:09 GMT)" (scheduled at 2025-11-04 09:36:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:36:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:36:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:36:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:36:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:36:09,601 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:36:11,368 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:36:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:36:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:15.535095+00:00 (in 4.004636 seconds) notificationworker stdout | 2025-11-04 09:36:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:21 GMT)" (scheduled at 2025-11-04 09:36:11.530064+00:00) notificationworker stdout | 2025-11-04 09:36:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:36:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 11, 530736), True, datetime.datetime(2025, 11, 4, 9, 36, 11, 530736), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:36:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:36:11,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:36:11,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:36:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:36:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:12.222828+00:00 (in 59.999619 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:36:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:12 GMT)" (scheduled at 2025-11-04 09:36:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:36:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:36:12,233 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:36:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:36:12,815 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:36:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:36:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:29.351216+00:00 (in 14.999574 seconds) quotatotalworker stdout | 2025-11-04 09:36:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:29 GMT)" (scheduled at 2025-11-04 09:36:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:36:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:36:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:36:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:36:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:36:15,315 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:36:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:36:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:17.479080+00:00 (in 1.998901 seconds) securityworker stdout | 2025-11-04 09:36:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:45 GMT)" (scheduled at 2025-11-04 09:36:15.479769+00:00) securityworker stdout | 2025-11-04 09:36:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:36:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:36:15,482 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:36:15,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:15,494 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:15,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 31, 15, 483544), 1, 3]) securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 31, 15, 483544), 1, 3]) securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:36:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:36:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:21.530064+00:00 (in 5.994520 seconds) notificationworker stdout | 2025-11-04 09:36:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:15 GMT)" (scheduled at 2025-11-04 09:36:15.535095+00:00) notificationworker stdout | 2025-11-04 09:36:15,536 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:36:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:36:15,697 [285] [DEBUG] [app] Starting request: urn:request:a5da9062-5531-43ee-84d1-7fe179694a36 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:15,701 [299] [DEBUG] [app] Starting request: urn:request:1d877dad-69ef-4766-bd12-9c19e00532e7 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:15,701 [299] [DEBUG] [app] Ending request: urn:request:1d877dad-69ef-4766-bd12-9c19e00532e7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1d877dad-69ef-4766-bd12-9c19e00532e7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:36:15,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:36:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:15,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:15,706 [288] [DEBUG] [app] Starting request: urn:request:6954a143-4e1b-4801-a7b2-714498fa13df (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:15,706 [288] [DEBUG] [app] Ending request: urn:request:6954a143-4e1b-4801-a7b2-714498fa13df (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6954a143-4e1b-4801-a7b2-714498fa13df', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:36:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:36:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:15,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:15,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:15,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:15,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:15,730 [285] [DEBUG] [app] Ending request: urn:request:a5da9062-5531-43ee-84d1-7fe179694a36 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a5da9062-5531-43ee-84d1-7fe179694a36', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:15,730 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) gunicorn-web stdout | 2025-11-04 09:36:15,783 [288] [DEBUG] [app] Starting request: urn:request:08fe802a-3192-4295-a4d7-6cabcb0fe22a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:15,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:15,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:15,788 [292] [DEBUG] [app] Starting request: urn:request:74b0fbed-a232-487a-9799-075c9a325d8a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:15,788 [292] [DEBUG] [app] Ending request: urn:request:74b0fbed-a232-487a-9799-075c9a325d8a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:74b0fbed-a232-487a-9799-075c9a325d8a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:36:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:15,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:15,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:15,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:15,792 [287] [DEBUG] [app] Starting request: urn:request:effe4b98-9e41-4ec2-9bf9-dbd0300e9322 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:15,793 [287] [DEBUG] [app] Ending request: urn:request:effe4b98-9e41-4ec2-9bf9-dbd0300e9322 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:effe4b98-9e41-4ec2-9bf9-dbd0300e9322', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:36:15,793 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:15,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:15,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:15,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:15,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:15,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:15,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:15,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:15,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:15,817 [288] [DEBUG] [app] Ending request: urn:request:08fe802a-3192-4295-a4d7-6cabcb0fe22a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:08fe802a-3192-4295-a4d7-6cabcb0fe22a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:15,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:15,817 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:36:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:36:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:27.122849+00:00 (in 9.999644 seconds) proxycacheblobworker stdout | 2025-11-04 09:36:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:27 GMT)" (scheduled at 2025-11-04 09:36:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:36:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:36:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 17, 123456), True, datetime.datetime(2025, 11, 4, 9, 36, 17, 123456), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:36:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:36:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:36:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:36:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:36:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:45.479769+00:00 (in 28.000221 seconds) securityworker stdout | 2025-11-04 09:36:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:47 GMT)" (scheduled at 2025-11-04 09:36:17.479080+00:00) securityworker stdout | 2025-11-04 09:36:17,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:36:17,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:36:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:36:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:36:17,498 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:36:17,498 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:47 GMT)" executed successfully builder stdout | 2025-11-04 09:36:18,119 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:36:18,131 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:36:18,131 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:36:18,141 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:36:18,251 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:36:18,924 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:36:20,238 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:36:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:36:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:51.226294+00:00 (in 29.999594 seconds) autopruneworker stdout | 2025-11-04 09:36:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:51 GMT)" (scheduled at 2025-11-04 09:36:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:36:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248921234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:36:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:36:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:36:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:36:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:36:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:32.435813+00:00 (in 11.000989 seconds) namespacegcworker stdout | 2025-11-04 09:36:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:21 GMT)" (scheduled at 2025-11-04 09:36:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:36:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:36:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 21, 435129), True, datetime.datetime(2025, 11, 4, 9, 36, 21, 435129), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:36:21,447 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:36:21,447 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:36:21,447 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:36:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:36:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:31.530064+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 09:36:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:31 GMT)" (scheduled at 2025-11-04 09:36:21.530064+00:00) notificationworker stdout | 2025-11-04 09:36:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:36:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 21, 530758), True, datetime.datetime(2025, 11, 4, 9, 36, 21, 530758), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:36:21,542 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:36:21,542 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:36:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:36:22,513 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:36:25,360 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:36:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:36:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:37.122849+00:00 (in 9.999576 seconds) proxycacheblobworker stdout | 2025-11-04 09:36:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:37 GMT)" (scheduled at 2025-11-04 09:36:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:36:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:36:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 27, 123559), True, datetime.datetime(2025, 11, 4, 9, 36, 27, 123559), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:36:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:36:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:36:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:36:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:36:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:57.222846+00:00 (in 29.999629 seconds) buildlogsarchiver stdout | 2025-11-04 09:36:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:57 GMT)" (scheduled at 2025-11-04 09:36:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:36:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 36, 27, 223410), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:36:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:36:27,238 [67] [DEBUG] [__main__] Archiving: 4708783e-59eb-4e42-b7a1-1923a4b4caff buildlogsarchiver stdout | 2025-11-04 09:36:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:36:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:36:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:36:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:36:27,592 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:36:28,141 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:36:28,154 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:36:28,154 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:36:28,164 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:36:28,304 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:36:28,955 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:36:29,339 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:36:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:36:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:44.351216+00:00 (in 14.999601 seconds) quotatotalworker stdout | 2025-11-04 09:36:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:44 GMT)" (scheduled at 2025-11-04 09:36:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:36:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:36:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:36:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:36:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:36:29,764 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:30,036 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:30,229 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:30,239 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:30,332 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:36:30,697 [285] [DEBUG] [app] Starting request: urn:request:07239422-989a-4dc5-9358-029316251a92 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:30,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:30,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:30,701 [292] [DEBUG] [app] Starting request: urn:request:4b9632fb-366d-4b47-bd59-0f34d1fccb8a (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:30,702 [292] [DEBUG] [app] Ending request: urn:request:4b9632fb-366d-4b47-bd59-0f34d1fccb8a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:4b9632fb-366d-4b47-bd59-0f34d1fccb8a', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:36:30,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:36:30,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:30,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:30,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:30,706 [287] [DEBUG] [app] Starting request: urn:request:8a7878a0-da5a-46da-98b2-d690f1122bad (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:30,706 [287] [DEBUG] [app] Ending request: urn:request:8a7878a0-da5a-46da-98b2-d690f1122bad (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8a7878a0-da5a-46da-98b2-d690f1122bad', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:36:30,707 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:30,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:36:30,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:30,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:30,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:30,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:30,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:30,725 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:30,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:30,731 [285] [DEBUG] [app] Ending request: urn:request:07239422-989a-4dc5-9358-029316251a92 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:07239422-989a-4dc5-9358-029316251a92', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:30,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:30,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:36:30,784 [288] [DEBUG] [app] Starting request: urn:request:1801a005-4da1-4896-9c11-fa3f2c6fe1a0 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:30,785 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:30,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:30,788 [292] [DEBUG] [app] Starting request: urn:request:04941eb4-e76a-4a39-a858-a6564b953f91 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:30,788 [292] [DEBUG] [app] Ending request: urn:request:04941eb4-e76a-4a39-a858-a6564b953f91 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:04941eb4-e76a-4a39-a858-a6564b953f91', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:36:30,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:30,789 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:30,790 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:30,792 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:30,792 [285] [DEBUG] [app] Starting request: urn:request:f8efd234-167c-4055-9d85-a4b3b50c80a9 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:30,793 [285] [DEBUG] [app] Ending request: urn:request:f8efd234-167c-4055-9d85-a4b3b50c80a9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f8efd234-167c-4055-9d85-a4b3b50c80a9', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:36:30,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:30,793 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:30,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:30,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:30,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:30,801 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:30,801 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:30,811 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:30,814 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:30,817 [288] [DEBUG] [app] Ending request: urn:request:1801a005-4da1-4896-9c11-fa3f2c6fe1a0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1801a005-4da1-4896-9c11-fa3f2c6fe1a0', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:30,817 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:30,818 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) notificationworker stdout | 2025-11-04 09:36:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:36:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:41.530064+00:00 (in 9.999553 seconds) notificationworker stdout | 2025-11-04 09:36:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:41 GMT)" (scheduled at 2025-11-04 09:36:31.530064+00:00) notificationworker stdout | 2025-11-04 09:36:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:36:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 31, 530783), True, datetime.datetime(2025, 11, 4, 9, 36, 31, 530783), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:36:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:36:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:36:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:41 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:36:31,597 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:36:32,394 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:36:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:36:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:21.434460+00:00 (in 48.998152 seconds) namespacegcworker stdout | 2025-11-04 09:36:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:32 GMT)" (scheduled at 2025-11-04 09:36:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:36:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:36:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:32 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:36:32,578 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:36:32,592 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:36:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:36:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:48.625192+00:00 (in 15.001825 seconds) exportactionlogsworker stdout | 2025-11-04 09:36:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:33 GMT)" (scheduled at 2025-11-04 09:36:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:36:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:36:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 33, 623624), True, datetime.datetime(2025, 11, 4, 9, 36, 33, 623624), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:36:33,635 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:36:33,635 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:36:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:36:33,870 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:36:33,870 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:36:33,972 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:36:34,748 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:36:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:36:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:38.437104+00:00 (in 3.000917 seconds) gcworker stdout | 2025-11-04 09:36:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:05 GMT)" (scheduled at 2025-11-04 09:36:35.435809+00:00) gcworker stdout | 2025-11-04 09:36:35,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039395436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:36:35,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:36:35,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:36:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:36:36,070 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:36:36,441 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:36:37,057 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:36:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:36:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:40.124445+00:00 (in 3.001173 seconds) proxycacheblobworker stdout | 2025-11-04 09:36:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:47 GMT)" (scheduled at 2025-11-04 09:36:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:36:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:36:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 37, 123569), True, datetime.datetime(2025, 11, 4, 9, 36, 37, 123569), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:36:37,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:36:37,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:36:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:36:37,485 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:36:38,164 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:36:38,177 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:36:38,177 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:36:38,187 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:36:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:36:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:05.435809+00:00 (in 26.998267 seconds) gcworker stdout | 2025-11-04 09:36:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:08 GMT)" (scheduled at 2025-11-04 09:36:38.437104+00:00) gcworker stdout | 2025-11-04 09:36:38,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:36:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248938449, None, 1, 0]) gcworker stdout | 2025-11-04 09:36:38,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:36:38,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:36:38,519 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,510 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,537 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,557 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,577 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,582 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,609 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,775 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:36:38,783 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:36:38,904 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:36:38,915 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:36:38,916 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:36:39,610 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:36:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:36:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:01.922827+00:00 (in 21.997750 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:36:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:39 GMT)" (scheduled at 2025-11-04 09:36:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:36:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:36:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:36:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:36:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:47.122849+00:00 (in 6.997983 seconds) proxycacheblobworker stdout | 2025-11-04 09:36:40,124 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:40 GMT)" (scheduled at 2025-11-04 09:36:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:36:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:36:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:36:41,376 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:36:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:36:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:51.530064+00:00 (in 9.999615 seconds) notificationworker stdout | 2025-11-04 09:36:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:51 GMT)" (scheduled at 2025-11-04 09:36:41.530064+00:00) notificationworker stdout | 2025-11-04 09:36:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:36:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 41, 530717), True, datetime.datetime(2025, 11, 4, 9, 36, 41, 530717), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:36:41,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:36:41,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:36:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:36:42,824 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:36:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:36:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:57.523915+00:00 (in 14.090622 seconds) repositorygcworker stdout | 2025-11-04 09:36:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:43 GMT)" (scheduled at 2025-11-04 09:36:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:36:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:36:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 43, 433602), True, datetime.datetime(2025, 11, 4, 9, 36, 43, 433602), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:36:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:36:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:36:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:36:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:36:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:59.351216+00:00 (in 14.999564 seconds) quotatotalworker stdout | 2025-11-04 09:36:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:59 GMT)" (scheduled at 2025-11-04 09:36:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:36:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:36:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:36:44,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:36:44,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:36:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:36:45,324 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:36:45,480 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:36:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:47.479080+00:00 (in 1.998862 seconds) securityworker stdout | 2025-11-04 09:36:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:15 GMT)" (scheduled at 2025-11-04 09:36:45.479769+00:00) securityworker stdout | 2025-11-04 09:36:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:36:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:36:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:36:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:45,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:45,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:45,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:45,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stdout | 2025-11-04 09:36:45,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:45,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 31, 45, 483922), 1, 3]) securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:45,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:36:45,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:45,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:45,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:45,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:45,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 31, 45, 483922), 1, 3]) securityworker stdout | 2025-11-04 09:36:45,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:45,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:45,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:45,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:36:45,697 [285] [DEBUG] [app] Starting request: urn:request:90d4a56c-cbc8-443c-9d4c-6fed3af2462d (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:45,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:45,701 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:45,702 [292] [DEBUG] [app] Starting request: urn:request:5f4692d5-634a-4f0e-abed-aa41dadaed09 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:45,702 [292] [DEBUG] [app] Ending request: urn:request:5f4692d5-634a-4f0e-abed-aa41dadaed09 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5f4692d5-634a-4f0e-abed-aa41dadaed09', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:36:45,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:36:45,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:45,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:45,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:45,706 [288] [DEBUG] [app] Starting request: urn:request:daaab91e-c43d-4cf5-81f9-24c6e5f04658 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:45,706 [288] [DEBUG] [app] Ending request: urn:request:daaab91e-c43d-4cf5-81f9-24c6e5f04658 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:daaab91e-c43d-4cf5-81f9-24c6e5f04658', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:36:45,707 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:36:45,707 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:45,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:45,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:45,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:45,715 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:45,715 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:45,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:45,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:45,731 [285] [DEBUG] [app] Ending request: urn:request:90d4a56c-cbc8-443c-9d4c-6fed3af2462d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:90d4a56c-cbc8-443c-9d4c-6fed3af2462d', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:45,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:45,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:36:45,784 [286] [DEBUG] [app] Starting request: urn:request:810f9d2d-30b0-4be0-9d90-0fd904d057a2 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:36:45,785 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:45,788 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:36:45,789 [298] [DEBUG] [app] Starting request: urn:request:f90b601a-d834-4fdf-8477-2e90673f65ff (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:36:45,789 [298] [DEBUG] [app] Ending request: urn:request:f90b601a-d834-4fdf-8477-2e90673f65ff (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f90b601a-d834-4fdf-8477-2e90673f65ff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:36:45,790 [298] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:45,790 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:45,790 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:36:45,792 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:36:45,793 [285] [DEBUG] [app] Starting request: urn:request:1aafce75-02f2-4c5f-b82e-f8e5ef4a26f7 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:36:45,793 [285] [DEBUG] [app] Ending request: urn:request:1aafce75-02f2-4c5f-b82e-f8e5ef4a26f7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1aafce75-02f2-4c5f-b82e-f8e5ef4a26f7', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:36:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:36:45,794 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:36:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:36:45,794 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:36:45,794 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:36:45,794 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:36:45,794 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:36:45,801 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:36:45,802 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:36:45,811 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:36:45,815 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:36:45,818 [286] [DEBUG] [app] Ending request: urn:request:810f9d2d-30b0-4be0-9d90-0fd904d057a2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:810f9d2d-30b0-4be0-9d90-0fd904d057a2', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:36:45,818 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:36:45,818 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:36:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:36:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.035) proxycacheblobworker stdout | 2025-11-04 09:36:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:36:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:57.122849+00:00 (in 9.999614 seconds) proxycacheblobworker stdout | 2025-11-04 09:36:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:57 GMT)" (scheduled at 2025-11-04 09:36:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:36:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:36:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 47, 123500), True, datetime.datetime(2025, 11, 4, 9, 36, 47, 123500), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:36:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:36:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:36:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:36:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:36:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:36:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:15.479769+00:00 (in 28.000171 seconds) securityworker stdout | 2025-11-04 09:36:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:17 GMT)" (scheduled at 2025-11-04 09:36:47.479080+00:00) securityworker stdout | 2025-11-04 09:36:47,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:36:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:36:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:36:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:36:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:36:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:36:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:47,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 31, 47, 483183), 1, 3]) securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:36:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 31, 47, 483183), 1, 3]) securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:36:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:36:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:17 GMT)" executed successfully builder stdout | 2025-11-04 09:36:48,187 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:36:48,201 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:36:48,201 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:36:48,211 [66] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:36:48,260 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:36:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:36:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:36:58.624878+00:00 (in 9.999218 seconds) exportactionlogsworker stdout | 2025-11-04 09:36:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:48 GMT)" (scheduled at 2025-11-04 09:36:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:36:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:36:48,626 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:36:48,932 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:36:50,247 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:36:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:36:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:21.226294+00:00 (in 29.999532 seconds) autopruneworker stdout | 2025-11-04 09:36:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:21 GMT)" (scheduled at 2025-11-04 09:36:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:36:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248951235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:36:51,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:36:51,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:36:51,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:36:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:36:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:01.530064+00:00 (in 9.999614 seconds) notificationworker stdout | 2025-11-04 09:36:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:01 GMT)" (scheduled at 2025-11-04 09:36:51.530064+00:00) notificationworker stdout | 2025-11-04 09:36:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:36:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 51, 530728), True, datetime.datetime(2025, 11, 4, 9, 36, 51, 530728), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:36:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:36:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:36:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:36:52,522 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:36:55,369 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:36:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:36:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:07.122849+00:00 (in 9.999569 seconds) proxycacheblobworker stdout | 2025-11-04 09:36:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:07 GMT)" (scheduled at 2025-11-04 09:36:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:36:57,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:36:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 36, 57, 123614), True, datetime.datetime(2025, 11, 4, 9, 36, 57, 123614), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:36:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:36:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:09.133645+00:00 (in 12.001455 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:36:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:57 GMT)" (scheduled at 2025-11-04 09:36:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:36:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:36:57,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:36:57,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:36:57,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:36:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:36:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:36:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:36:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:36:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:27.222846+00:00 (in 29.999592 seconds) buildlogsarchiver stdout | 2025-11-04 09:36:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:27 GMT)" (scheduled at 2025-11-04 09:36:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:36:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 36, 57, 223507), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:36:57,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:36:57,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:36:57,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:36:57,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:36:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:36:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:36:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:43.432867+00:00 (in 45.908493 seconds) repositorygcworker stdout | 2025-11-04 09:36:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:57 GMT)" (scheduled at 2025-11-04 09:36:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:36:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:36:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:37:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:36:57,601 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:36:58,211 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:36:58,224 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:36:58,224 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:36:58,234 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:36:58,313 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:36:58,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:36:58,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:33.622985+00:00 (in 34.997386 seconds) exportactionlogsworker stdout | 2025-11-04 09:36:58,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:41:58 GMT)" (scheduled at 2025-11-04 09:36:58.624878+00:00) exportactionlogsworker stdout | 2025-11-04 09:36:58,626 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 36, 58, 625869), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:36:58,638 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 36, 58, 625869), True, datetime.datetime(2025, 11, 4, 9, 36, 58, 625869), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:36:58,642 [71] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 36, 58, 625869), True, datetime.datetime(2025, 11, 4, 9, 36, 58, 625869), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 36, 58, 625869), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:36:58,645 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:36:58,645 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:41:58 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:36:58,964 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:36:59,348 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:36:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:36:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:14.351216+00:00 (in 14.999587 seconds) quotatotalworker stdout | 2025-11-04 09:36:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:14 GMT)" (scheduled at 2025-11-04 09:36:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:36:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:36:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:36:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:36:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:36:59,774 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:00,052 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:00,242 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:00,253 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:00,347 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:00,697 [285] [DEBUG] [app] Starting request: urn:request:8ce39201-7afd-4331-8ec2-581004f31309 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:00,698 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:00,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:00,701 [292] [DEBUG] [app] Starting request: urn:request:63f92ff5-123f-4d36-b049-f59c04008456 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:00,702 [292] [DEBUG] [app] Ending request: urn:request:63f92ff5-123f-4d36-b049-f59c04008456 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:63f92ff5-123f-4d36-b049-f59c04008456', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:37:00,702 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:37:00,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:00,703 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:00,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:00,706 [287] [DEBUG] [app] Starting request: urn:request:999f51d5-0846-46cb-98a0-531aac0cb4b4 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:00,706 [287] [DEBUG] [app] Ending request: urn:request:999f51d5-0846-46cb-98a0-531aac0cb4b4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:999f51d5-0846-46cb-98a0-531aac0cb4b4', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:37:00,706 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:37:00,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:00,707 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:00,707 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:00,707 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:00,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:00,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:00,724 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:00,728 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:00,731 [285] [DEBUG] [app] Ending request: urn:request:8ce39201-7afd-4331-8ec2-581004f31309 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8ce39201-7afd-4331-8ec2-581004f31309', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:00,731 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:37:00,731 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:37:00,784 [285] [DEBUG] [app] Starting request: urn:request:ca20d31d-9279-43e8-bf80-d995ad36874c (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:00,785 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:00,785 [292] [DEBUG] [app] Starting request: urn:request:d6df96e3-60d7-4126-a5e4-057269f2f76d (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:00,786 [292] [DEBUG] [app] Ending request: urn:request:d6df96e3-60d7-4126-a5e4-057269f2f76d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d6df96e3-60d7-4126-a5e4-057269f2f76d', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:37:00,786 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:37:00,786 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:00,787 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:00,789 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:00,789 [287] [DEBUG] [app] Starting request: urn:request:47092bd3-fe77-4380-8a02-c582bc396800 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:00,790 [287] [DEBUG] [app] Ending request: urn:request:47092bd3-fe77-4380-8a02-c582bc396800 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:47092bd3-fe77-4380-8a02-c582bc396800', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.000) gunicorn-web stdout | 2025-11-04 09:37:00,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:37:00,790 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:00,790 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:00,790 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:00,790 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:00,798 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:00,798 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:00,808 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:00,811 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:00,814 [285] [DEBUG] [app] Ending request: urn:request:ca20d31d-9279-43e8-bf80-d995ad36874c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ca20d31d-9279-43e8-bf80-d995ad36874c', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:00,814 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:37:00,814 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.031) notificationworker stdout | 2025-11-04 09:37:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:37:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:11.530064+00:00 (in 9.999598 seconds) notificationworker stdout | 2025-11-04 09:37:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:11 GMT)" (scheduled at 2025-11-04 09:37:01.530064+00:00) notificationworker stdout | 2025-11-04 09:37:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:37:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 1, 530773), True, datetime.datetime(2025, 11, 4, 9, 37, 1, 530773), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:37:01,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:37:01,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:37:01,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:11 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:37:01,605 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:37:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:37:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:21.924304+00:00 (in 20.001097 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:37:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:01 GMT)" (scheduled at 2025-11-04 09:37:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:37:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:37:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 1, 923461), True, datetime.datetime(2025, 11, 4, 9, 37, 1, 923461), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:37:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:37:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:37:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:37:02,403 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:37:02,587 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:37:02,601 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:37:03,878 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:37:03,878 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:37:03,980 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:04,758 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:37:05,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:37:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:08.437104+00:00 (in 3.000913 seconds) gcworker stdout | 2025-11-04 09:37:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:35 GMT)" (scheduled at 2025-11-04 09:37:05.435809+00:00) gcworker stdout | 2025-11-04 09:37:05,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039425436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:37:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:37:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:37:05,451 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:37:06,079 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:37:06,450 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:37:07,067 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:37:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:37:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:17.122849+00:00 (in 9.999570 seconds) proxycacheblobworker stdout | 2025-11-04 09:37:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:17 GMT)" (scheduled at 2025-11-04 09:37:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:37:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:37:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 7, 123582), True, datetime.datetime(2025, 11, 4, 9, 37, 7, 123582), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:37:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:37:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:37:07,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:37:07,495 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:08,234 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:37:08,248 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:37:08,248 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:37:08,258 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:37:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:37:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:35.435809+00:00 (in 26.998292 seconds) gcworker stdout | 2025-11-04 09:37:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:38 GMT)" (scheduled at 2025-11-04 09:37:08.437104+00:00) gcworker stdout | 2025-11-04 09:37:08,438 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:37:08,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248968449, None, 1, 0]) gcworker stdout | 2025-11-04 09:37:08,453 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:37:08,453 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:38 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:37:08,532 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,541 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,551 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,568 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,593 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,593 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,620 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,795 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:08,796 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:37:08,914 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:37:08,925 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:37:08,926 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:37:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:37:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:57.131795+00:00 (in 47.997716 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:37:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:09 GMT)" (scheduled at 2025-11-04 09:37:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:37:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:37:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:37:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:37:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:37:09,619 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:37:11,385 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:37:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:37:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:15.535095+00:00 (in 4.004612 seconds) notificationworker stdout | 2025-11-04 09:37:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:21 GMT)" (scheduled at 2025-11-04 09:37:11.530064+00:00) notificationworker stdout | 2025-11-04 09:37:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:37:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 11, 530731), True, datetime.datetime(2025, 11, 4, 9, 37, 11, 530731), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:37:11,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:37:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:37:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:37:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:37:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:12.222828+00:00 (in 59.999578 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:37:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:12 GMT)" (scheduled at 2025-11-04 09:37:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:37:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:37:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:37:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:37:12,833 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:37:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:37:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:29.351216+00:00 (in 14.999612 seconds) quotatotalworker stdout | 2025-11-04 09:37:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:29 GMT)" (scheduled at 2025-11-04 09:37:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:37:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:37:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:37:14,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:37:14,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:37:15,333 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:37:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:37:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:17.479080+00:00 (in 1.998875 seconds) securityworker stdout | 2025-11-04 09:37:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:45 GMT)" (scheduled at 2025-11-04 09:37:15.479769+00:00) securityworker stdout | 2025-11-04 09:37:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:37:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:37:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:37:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:15,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:15,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:15,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:37:15,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:37:15,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:15,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 32, 15, 484141), 1, 3]) securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 32, 15, 484141), 1, 3]) securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:15,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:37:15,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:37:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:37:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:21.530064+00:00 (in 5.994313 seconds) notificationworker stdout | 2025-11-04 09:37:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:15 GMT)" (scheduled at 2025-11-04 09:37:15.535095+00:00) notificationworker stdout | 2025-11-04 09:37:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:37:15,536 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:37:15,696 [285] [DEBUG] [app] Starting request: urn:request:d6af42bf-ec7f-4a9f-aadf-d0ae14662600 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:15,697 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:15,700 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:15,701 [299] [DEBUG] [app] Starting request: urn:request:eecdc534-6a99-4cdf-b7a3-6ef71525cc47 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:15,701 [299] [DEBUG] [app] Ending request: urn:request:eecdc534-6a99-4cdf-b7a3-6ef71525cc47 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:eecdc534-6a99-4cdf-b7a3-6ef71525cc47', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:37:15,701 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:37:15,702 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:15,702 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:15,705 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:15,705 [288] [DEBUG] [app] Starting request: urn:request:87f8cd78-08be-4514-87c2-1aea932fe303 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:15,706 [288] [DEBUG] [app] Ending request: urn:request:87f8cd78-08be-4514-87c2-1aea932fe303 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:87f8cd78-08be-4514-87c2-1aea932fe303', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:37:15,706 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:37:15,706 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:15,706 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:15,706 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:15,706 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:15,714 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:15,714 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:15,723 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:15,727 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:15,729 [285] [DEBUG] [app] Ending request: urn:request:d6af42bf-ec7f-4a9f-aadf-d0ae14662600 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d6af42bf-ec7f-4a9f-aadf-d0ae14662600', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:15,730 [285] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:37:15,730 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:37:15,783 [288] [DEBUG] [app] Starting request: urn:request:08fb45dc-62f0-410b-b23a-93f27f514199 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:15,784 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:15,787 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:15,787 [292] [DEBUG] [app] Starting request: urn:request:d3b3b8e3-81d3-4e6d-afd0-71ef937b5648 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:15,788 [292] [DEBUG] [app] Ending request: urn:request:d3b3b8e3-81d3-4e6d-afd0-71ef937b5648 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d3b3b8e3-81d3-4e6d-afd0-71ef937b5648', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:37:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:37:15,788 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:15,789 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:15,791 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:15,792 [285] [DEBUG] [app] Starting request: urn:request:ae84b7f9-cef3-4524-a74d-8172d81f47da (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:15,792 [285] [DEBUG] [app] Ending request: urn:request:ae84b7f9-cef3-4524-a74d-8172d81f47da (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ae84b7f9-cef3-4524-a74d-8172d81f47da', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:37:15,792 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:37:15,792 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:15,793 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:15,793 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:15,793 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:15,800 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:15,800 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:15,810 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:15,813 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:15,816 [288] [DEBUG] [app] Ending request: urn:request:08fb45dc-62f0-410b-b23a-93f27f514199 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:08fb45dc-62f0-410b-b23a-93f27f514199', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:15,816 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:37:15,816 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:37:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:37:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:27.122849+00:00 (in 9.999612 seconds) proxycacheblobworker stdout | 2025-11-04 09:37:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:27 GMT)" (scheduled at 2025-11-04 09:37:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:37:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:37:17,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 17, 123506), True, datetime.datetime(2025, 11, 4, 9, 37, 17, 123506), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:37:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:37:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:37:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:37:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:37:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:45.479769+00:00 (in 28.000304 seconds) securityworker stdout | 2025-11-04 09:37:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:47 GMT)" (scheduled at 2025-11-04 09:37:17.479080+00:00) securityworker stdout | 2025-11-04 09:37:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:37:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:37:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:37:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:37:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:37:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:47 GMT)" executed successfully builder stdout | 2025-11-04 09:37:18,258 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue securityscanningnotificationworker stdout | 2025-11-04 09:37:18,269 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:18,271 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:37:18,271 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:37:18,281 [66] [DEBUG] [data.database] Disconnecting from database. reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:37:18,941 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:37:20,255 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:37:21,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:37:21,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:51.226294+00:00 (in 29.999532 seconds) autopruneworker stdout | 2025-11-04 09:37:21,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:51 GMT)" (scheduled at 2025-11-04 09:37:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:37:21,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762248981234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:37:21,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:37:21,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:37:21,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:37:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:37:21,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:32.435813+00:00 (in 11.000807 seconds) namespacegcworker stdout | 2025-11-04 09:37:21,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:21 GMT)" (scheduled at 2025-11-04 09:37:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:37:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:37:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 21, 435279), True, datetime.datetime(2025, 11, 4, 9, 37, 21, 435279), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:37:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:37:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:37:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:37:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:37:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:31.530064+00:00 (in 9.999645 seconds) notificationworker stdout | 2025-11-04 09:37:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:31 GMT)" (scheduled at 2025-11-04 09:37:21.530064+00:00) notificationworker stdout | 2025-11-04 09:37:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:37:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 21, 530702), True, datetime.datetime(2025, 11, 4, 9, 37, 21, 530702), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:37:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:37:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:37:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:31 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:37:21,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:37:21,924 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:39.924652+00:00 (in 17.999915 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:37:21,924 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:42:21 GMT)" (scheduled at 2025-11-04 09:37:21.924304+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:37:21,925 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 37, 21, 925010), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:37:21,937 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 37, 21, 925010), True, datetime.datetime(2025, 11, 4, 9, 37, 21, 925010), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:37:21,940 [101] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 37, 21, 925010), True, datetime.datetime(2025, 11, 4, 9, 37, 21, 925010), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 37, 21, 925010), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:37:21,943 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:37:21,944 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:42:21 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:37:22,530 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:37:25,378 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:37:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:37:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:37.122849+00:00 (in 9.999589 seconds) proxycacheblobworker stdout | 2025-11-04 09:37:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:37 GMT)" (scheduled at 2025-11-04 09:37:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:37:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:37:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 27, 123593), True, datetime.datetime(2025, 11, 4, 9, 37, 27, 123593), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:37:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:37:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:37:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:37:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:37:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:57.222846+00:00 (in 29.999621 seconds) buildlogsarchiver stdout | 2025-11-04 09:37:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:57 GMT)" (scheduled at 2025-11-04 09:37:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:37:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 37, 27, 223426), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:37:27,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:37:27,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:37:27,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:37:27,246 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:37:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:37:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:37:27,610 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:28,281 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:37:28,294 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:37:28,295 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:37:28,304 [66] [DEBUG] [data.database] Disconnecting from database. logrotateworker stdout | 2025-11-04 09:37:28,321 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: storagereplication stdout | 2025-11-04 09:37:28,972 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:37:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:37:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:44.351216+00:00 (in 14.999586 seconds) quotatotalworker stdout | 2025-11-04 09:37:29,352 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:44 GMT)" (scheduled at 2025-11-04 09:37:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:37:29,352 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:37:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) teamsyncworker stdout | 2025-11-04 09:37:29,356 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:37:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:37:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:37:29,784 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:30,065 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:30,256 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:30,267 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:30,361 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:30,697 [286] [DEBUG] [app] Starting request: urn:request:f017ae69-2058-4dfa-95e8-45ad546cd245 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:30,698 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:30,700 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:30,701 [299] [DEBUG] [app] Starting request: urn:request:76d5dfaa-b5df-4938-8a89-f46d1f4586ec (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:30,702 [299] [DEBUG] [app] Ending request: urn:request:76d5dfaa-b5df-4938-8a89-f46d1f4586ec (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:76d5dfaa-b5df-4938-8a89-f46d1f4586ec', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:37:30,702 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:37:30,702 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:30,703 [286] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:30,705 [286] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:30,706 [285] [DEBUG] [app] Starting request: urn:request:af9ed6b0-8ddb-4404-966e-75e85493005c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:30,706 [285] [DEBUG] [app] Ending request: urn:request:af9ed6b0-8ddb-4404-966e-75e85493005c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:af9ed6b0-8ddb-4404-966e-75e85493005c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:37:30,706 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:37:30,706 [286] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:30,707 [286] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:30,707 [286] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:30,707 [286] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:30,714 [286] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:30,714 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:30,724 [286] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:30,727 [286] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:30,730 [286] [DEBUG] [app] Ending request: urn:request:f017ae69-2058-4dfa-95e8-45ad546cd245 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f017ae69-2058-4dfa-95e8-45ad546cd245', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:30,730 [286] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:37:30,730 [286] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:37:30,784 [287] [DEBUG] [app] Starting request: urn:request:9e81a008-2a3c-467c-9c47-858df35e4057 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:30,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:30,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:30,788 [299] [DEBUG] [app] Starting request: urn:request:a3a32a62-1477-40d2-9a4e-4a3913e8f310 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:30,788 [299] [DEBUG] [app] Ending request: urn:request:a3a32a62-1477-40d2-9a4e-4a3913e8f310 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a3a32a62-1477-40d2-9a4e-4a3913e8f310', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:37:30,789 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:37:30,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:30,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:30,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:30,792 [285] [DEBUG] [app] Starting request: urn:request:23169d44-332b-4735-a4ca-e60b2d614674 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:30,792 [285] [DEBUG] [app] Ending request: urn:request:23169d44-332b-4735-a4ca-e60b2d614674 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:23169d44-332b-4735-a4ca-e60b2d614674', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:37:30,793 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:37:30,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:37:30,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:30,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:30,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:30,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:30,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:30,811 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:30,815 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:30,818 [287] [DEBUG] [app] Ending request: urn:request:9e81a008-2a3c-467c-9c47-858df35e4057 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9e81a008-2a3c-467c-9c47-858df35e4057', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:30,818 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:37:30,818 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) notificationworker stdout | 2025-11-04 09:37:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:37:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:41.530064+00:00 (in 9.999603 seconds) notificationworker stdout | 2025-11-04 09:37:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:41 GMT)" (scheduled at 2025-11-04 09:37:31.530064+00:00) notificationworker stdout | 2025-11-04 09:37:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:37:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 31, 530746), True, datetime.datetime(2025, 11, 4, 9, 37, 31, 530746), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:37:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:37:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:37:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:41 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:37:31,614 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:37:32,411 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:37:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:37:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:08.435432+00:00 (in 35.999238 seconds) namespacegcworker stdout | 2025-11-04 09:37:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:32 GMT)" (scheduled at 2025-11-04 09:37:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:37:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:37:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:32 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:37:32,595 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:37:32,609 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:37:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:37:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:48.625192+00:00 (in 15.001749 seconds) exportactionlogsworker stdout | 2025-11-04 09:37:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:33 GMT)" (scheduled at 2025-11-04 09:37:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:37:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:37:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 33, 623765), True, datetime.datetime(2025, 11, 4, 9, 37, 33, 623765), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:37:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:37:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:37:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:37:33,887 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:37:33,887 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:37:33,989 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:34,767 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:37:35,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:37:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:38.437104+00:00 (in 3.000887 seconds) gcworker stdout | 2025-11-04 09:37:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:05 GMT)" (scheduled at 2025-11-04 09:37:35.435809+00:00) gcworker stdout | 2025-11-04 09:37:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039455436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:37:35,449 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:37:35,449 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:37:35,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:05 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:37:36,088 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:37:36,458 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:37:37,076 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:37:37,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:37:37,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:40.124445+00:00 (in 3.001194 seconds) proxycacheblobworker stdout | 2025-11-04 09:37:37,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:47 GMT)" (scheduled at 2025-11-04 09:37:37.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:37:37,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:37:37,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 37, 123506), True, datetime.datetime(2025, 11, 4, 9, 37, 37, 123506), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:37:37,135 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:37:37,135 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:37:37,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:47 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:37:37,503 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:38,305 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:37:38,318 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:37:38,318 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:37:38,328 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:37:38,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:37:38,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:05.435809+00:00 (in 26.998258 seconds) gcworker stdout | 2025-11-04 09:37:38,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:08 GMT)" (scheduled at 2025-11-04 09:37:38.437104+00:00) gcworker stdout | 2025-11-04 09:37:38,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:37:38,449 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762248998448, None, 1, 0]) gcworker stdout | 2025-11-04 09:37:38,452 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:37:38,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:37:38,547 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,555 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,564 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,579 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,602 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,607 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,631 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,811 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:37:38,811 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:37:38,924 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:37:38,940 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:37:38,941 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:37:39,627 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:37:39,924 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:37:39,925 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:01.922827+00:00 (in 21.997750 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:37:39,925 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:39 GMT)" (scheduled at 2025-11-04 09:37:39.924652+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:37:39,925 [101] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:37:39,925 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:39 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:37:40,124 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:37:40,124 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:47.122849+00:00 (in 6.997968 seconds) proxycacheblobworker stdout | 2025-11-04 09:37:40,125 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:40 GMT)" (scheduled at 2025-11-04 09:37:40.124445+00:00) proxycacheblobworker stdout | 2025-11-04 09:37:40,125 [91] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:37:40,125 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:40 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:37:41,394 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:37:41,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:37:41,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:51.530064+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 09:37:41,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:51 GMT)" (scheduled at 2025-11-04 09:37:41.530064+00:00) notificationworker stdout | 2025-11-04 09:37:41,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:37:41,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 41, 530745), True, datetime.datetime(2025, 11, 4, 9, 37, 41, 530745), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:37:41,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:37:41,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:37:41,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:51 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:37:42,843 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:37:43,433 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:37:43,433 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:57.523915+00:00 (in 14.090640 seconds) repositorygcworker stdout | 2025-11-04 09:37:43,433 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:43 GMT)" (scheduled at 2025-11-04 09:37:43.432867+00:00) repositorygcworker stdout | 2025-11-04 09:37:43,433 [100] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:37:43,434 [100] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 43, 433558), True, datetime.datetime(2025, 11, 4, 9, 37, 43, 433558), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:37:43,446 [100] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:37:43,446 [100] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:37:43,446 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:43 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:37:44,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:37:44,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:59.351216+00:00 (in 14.999590 seconds) quotatotalworker stdout | 2025-11-04 09:37:44,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:59 GMT)" (scheduled at 2025-11-04 09:37:44.351216+00:00) quotatotalworker stdout | 2025-11-04 09:37:44,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:37:44,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:37:44,363 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:37:44,363 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:37:59 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:37:45,342 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:37:45,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:37:45,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:47.479080+00:00 (in 1.998933 seconds) securityworker stdout | 2025-11-04 09:37:45,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:15 GMT)" (scheduled at 2025-11-04 09:37:45.479769+00:00) securityworker stdout | 2025-11-04 09:37:45,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:37:45,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:37:45,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:37:45,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:45,495 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:45,495 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:45,495 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:37:45,495 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:45,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:45,499 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:45,500 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 32, 45, 484088), 1, 3]) securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:37:45,503 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:45,504 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 32, 45, 484088), 1, 3]) securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:45,507 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:45 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:45,507 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:37:45,696 [288] [DEBUG] [app] Starting request: urn:request:6a6f0381-e9c1-4281-8d78-5626fdfd0708 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:45,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:45,699 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:45,700 [292] [DEBUG] [app] Starting request: urn:request:3fd30a2d-cdb8-4822-9981-d780835db6be (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:45,701 [292] [DEBUG] [app] Ending request: urn:request:3fd30a2d-cdb8-4822-9981-d780835db6be (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3fd30a2d-cdb8-4822-9981-d780835db6be', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:37:45,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:37:45,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:45,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:45,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:45,704 [285] [DEBUG] [app] Starting request: urn:request:6b2b2112-5fdf-4868-bf5b-3b2ad6c0dd04 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:45,705 [285] [DEBUG] [app] Ending request: urn:request:6b2b2112-5fdf-4868-bf5b-3b2ad6c0dd04 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6b2b2112-5fdf-4868-bf5b-3b2ad6c0dd04', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:37:45,705 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:37:45,705 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:45,705 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:45,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:45,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:45,713 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:45,713 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:45,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:45,726 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:45,729 [288] [DEBUG] [app] Ending request: urn:request:6a6f0381-e9c1-4281-8d78-5626fdfd0708 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6a6f0381-e9c1-4281-8d78-5626fdfd0708', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:45,729 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:37:45,729 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) gunicorn-web stdout | 2025-11-04 09:37:45,784 [287] [DEBUG] [app] Starting request: urn:request:09e4cf15-5efc-4e40-b74d-bf5419364812 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:37:45,785 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:45,787 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:37:45,788 [292] [DEBUG] [app] Starting request: urn:request:54f86c1b-1a28-409f-adf0-975838c9de28 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:37:45,788 [292] [DEBUG] [app] Ending request: urn:request:54f86c1b-1a28-409f-adf0-975838c9de28 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:54f86c1b-1a28-409f-adf0-975838c9de28', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:37:45,789 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:45 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:37:45,789 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:45,789 [287] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:37:45,792 [287] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:37:45,792 [288] [DEBUG] [app] Starting request: urn:request:ddead86b-e8af-48f5-9755-1dc4e303e576 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:37:45,792 [288] [DEBUG] [app] Ending request: urn:request:ddead86b-e8af-48f5-9755-1dc4e303e576 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ddead86b-e8af-48f5-9755-1dc4e303e576', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:37:45 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:37:45,793 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:37:45 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:37:45,793 [287] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:37:45,793 [287] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:37:45,793 [287] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:37:45,793 [287] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:37:45,801 [287] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:37:45,801 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:37:45,810 [287] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:37:45,814 [287] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:37:45,817 [287] [DEBUG] [app] Ending request: urn:request:09e4cf15-5efc-4e40-b74d-bf5419364812 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:09e4cf15-5efc-4e40-b74d-bf5419364812', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:37:45,817 [287] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:37:45,817 [287] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:37:45 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:37:45 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:37:47,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:37:47,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:37:57.122849+00:00 (in 9.999613 seconds) proxycacheblobworker stdout | 2025-11-04 09:37:47,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:57 GMT)" (scheduled at 2025-11-04 09:37:47.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:37:47,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:37:47,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 47, 123568), True, datetime.datetime(2025, 11, 4, 9, 37, 47, 123568), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:37:47,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:37:47,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:37:47,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:37:57 GMT)" executed successfully securityworker stdout | 2025-11-04 09:37:47,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:37:47,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:15.479769+00:00 (in 28.000263 seconds) securityworker stdout | 2025-11-04 09:37:47,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:17 GMT)" (scheduled at 2025-11-04 09:37:47.479080+00:00) securityworker stdout | 2025-11-04 09:37:47,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:37:47,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:37:47,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:37:47,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:37:47,494 [102] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:47,497 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:47,497 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:47,497 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:37:47,497 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:47,498 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:37:47,501 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:47,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:37:47,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:47,502 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 32, 47, 483069), 1, 3]) securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:37:47,505 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:37:47,506 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 32, 47, 483069), 1, 3]) securityworker stdout | 2025-11-04 09:37:47,509 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:47,509 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:37:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:37:47,509 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:37:47,509 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:37:47,509 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:37:47,509 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:47,510 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:37:47 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:37:47,510 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:17 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:37:48,278 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:48,328 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:37:48,341 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:37:48,341 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:37:48,351 [66] [DEBUG] [data.database] Disconnecting from database. reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:37:48,625 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:37:48,625 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:33.622985+00:00 (in 44.997354 seconds) exportactionlogsworker stdout | 2025-11-04 09:37:48,625 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:48 GMT)" (scheduled at 2025-11-04 09:37:48.625192+00:00) exportactionlogsworker stdout | 2025-11-04 09:37:48,625 [71] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:37:48,625 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:48 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:37:48,950 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:37:50,264 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:37:51,226 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:37:51,226 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:21.226294+00:00 (in 29.999532 seconds) autopruneworker stdout | 2025-11-04 09:37:51,226 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:21 GMT)" (scheduled at 2025-11-04 09:37:51.226294+00:00) autopruneworker stdout | 2025-11-04 09:37:51,235 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762249011234, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:37:51,239 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:37:51,239 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:37:51,239 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:37:51,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:37:51,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:01.530064+00:00 (in 9.999558 seconds) notificationworker stdout | 2025-11-04 09:37:51,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:01 GMT)" (scheduled at 2025-11-04 09:37:51.530064+00:00) notificationworker stdout | 2025-11-04 09:37:51,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:37:51,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 51, 530773), True, datetime.datetime(2025, 11, 4, 9, 37, 51, 530773), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:37:51,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:37:51,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:37:51,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:01 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:37:52,539 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:37:55,396 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:37:57,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:37:57,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:07.122849+00:00 (in 9.999046 seconds) proxycacheblobworker stdout | 2025-11-04 09:37:57,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:07 GMT)" (scheduled at 2025-11-04 09:37:57.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:37:57,124 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:37:57,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 37, 57, 124077), True, datetime.datetime(2025, 11, 4, 9, 37, 57, 124077), 0, 'proxycacheblob/%', 50, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:37:57,131 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:37:57,132 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:09.133645+00:00 (in 12.001487 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:37:57,132 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:57 GMT)" (scheduled at 2025-11-04 09:37:57.131795+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:37:57,132 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:37:57,137 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:37:57,137 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:37:57,137 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:07 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:37:57,143 [79] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:37:57,143 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:37:57,143 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:57 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:37:57,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:37:57,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:27.222846+00:00 (in 29.999592 seconds) buildlogsarchiver stdout | 2025-11-04 09:37:57,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:27 GMT)" (scheduled at 2025-11-04 09:37:57.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:37:57,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 37, 57, 223465), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:37:57,235 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:37:57,238 [67] [DEBUG] [__main__] Archiving: 295a8c3b-9ccf-40ed-a879-0dc20a330d9d buildlogsarchiver stdout | 2025-11-04 09:37:57,239 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:37:57,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:37:57,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:27 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:37:57,524 [100] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:37:57,524 [100] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:43.432867+00:00 (in 45.908497 seconds) repositorygcworker stdout | 2025-11-04 09:37:57,524 [100] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:57 GMT)" (scheduled at 2025-11-04 09:37:57.523915+00:00) repositorygcworker stdout | 2025-11-04 09:37:57,524 [100] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:37:57,524 [100] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:38:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:37:57,619 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:37:58,330 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:37:58,351 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:37:58,364 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:37:58,364 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:37:58,374 [66] [DEBUG] [data.database] Disconnecting from database. storagereplication stdout | 2025-11-04 09:37:58,981 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:37:59,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:37:59,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:14.351216+00:00 (in 14.999528 seconds) quotatotalworker stdout | 2025-11-04 09:37:59,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:38:14 GMT)" (scheduled at 2025-11-04 09:37:59.351216+00:00) quotatotalworker stdout | 2025-11-04 09:37:59,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:37:59,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:37:59,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:37:59,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:38:14 GMT)" executed successfully teamsyncworker stdout | 2025-11-04 09:37:59,365 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:37:59,795 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:00,078 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:00,269 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:00,280 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:00,376 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:00,696 [288] [DEBUG] [app] Starting request: urn:request:bb8d87fe-560a-4184-929e-b6308d6cda64 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:38:00,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:00,699 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:38:00,700 [297] [DEBUG] [app] Starting request: urn:request:5c781c62-9efd-472f-939d-3d2cd65d1cff (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:38:00,701 [297] [DEBUG] [app] Ending request: urn:request:5c781c62-9efd-472f-939d-3d2cd65d1cff (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5c781c62-9efd-472f-939d-3d2cd65d1cff', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:38:00,701 [297] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:38:00,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:00,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:00,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:38:00,705 [285] [DEBUG] [app] Starting request: urn:request:af872bd1-68ab-4886-97e8-cbcf985d8f6c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:38:00,705 [285] [DEBUG] [app] Ending request: urn:request:af872bd1-68ab-4886-97e8-cbcf985d8f6c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:af872bd1-68ab-4886-97e8-cbcf985d8f6c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:38:00,705 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:38:00,705 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:00,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:38:00,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:38:00,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:38:00,713 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:38:00,713 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:38:00,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:38:00,726 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:38:00,729 [288] [DEBUG] [app] Ending request: urn:request:bb8d87fe-560a-4184-929e-b6308d6cda64 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bb8d87fe-560a-4184-929e-b6308d6cda64', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:38:00,729 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:38:00,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:38:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:38:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:38:00,783 [288] [DEBUG] [app] Starting request: urn:request:9b01817e-aaff-42fb-9aab-a62a1dba8f92 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:38:00,784 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:38:00,785 [292] [DEBUG] [app] Starting request: urn:request:a2191ab3-0667-47b7-8857-982df90cddcb (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:38:00,785 [292] [DEBUG] [app] Ending request: urn:request:a2191ab3-0667-47b7-8857-982df90cddcb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a2191ab3-0667-47b7-8857-982df90cddcb', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:38:00,785 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:00 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:38:00,785 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:00,786 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:00,788 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:38:00,789 [287] [DEBUG] [app] Starting request: urn:request:cb91de92-080c-42ec-947e-f18828cc3a2c (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:38:00,789 [287] [DEBUG] [app] Ending request: urn:request:cb91de92-080c-42ec-947e-f18828cc3a2c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cb91de92-080c-42ec-947e-f18828cc3a2c', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:38:00,790 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:00 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:00 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:38:00,790 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:00,790 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:38:00,790 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:38:00,790 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:38:00,797 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:38:00,797 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:38:00,806 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:38:00,810 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:38:00,813 [288] [DEBUG] [app] Ending request: urn:request:9b01817e-aaff-42fb-9aab-a62a1dba8f92 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9b01817e-aaff-42fb-9aab-a62a1dba8f92', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:38:00,813 [288] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:38:00,813 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:38:00 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:38:00 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.031 47 0.030) notificationworker stdout | 2025-11-04 09:38:01,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:38:01,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:11.530064+00:00 (in 9.999606 seconds) notificationworker stdout | 2025-11-04 09:38:01,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:11 GMT)" (scheduled at 2025-11-04 09:38:01.530064+00:00) notificationworker stdout | 2025-11-04 09:38:01,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:38:01,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 1, 530740), True, datetime.datetime(2025, 11, 4, 9, 38, 1, 530740), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:38:01,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:38:01,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:38:01,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:11 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:38:01,623 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:38:01,923 [101] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:38:01,923 [101] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:39.924652+00:00 (in 38.001433 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:38:01,923 [101] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:01 GMT)" (scheduled at 2025-11-04 09:38:01.922827+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:38:01,923 [101] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:38:01,924 [101] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 1, 923482), True, datetime.datetime(2025, 11, 4, 9, 38, 1, 923482), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:38:01,935 [101] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:38:01,935 [101] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:38:01,935 [101] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:01 GMT)" executed successfully gcworker stdout | 2025-11-04 09:38:02,420 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:38:02,615 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:38:02,618 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:38:03,895 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:38:03,895 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:38:03,997 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:38:04,777 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:38:05,435 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:38:05,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:08.437104+00:00 (in 3.000921 seconds) gcworker stdout | 2025-11-04 09:38:05,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:35 GMT)" (scheduled at 2025-11-04 09:38:05.435809+00:00) gcworker stdout | 2025-11-04 09:38:05,436 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039485436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:38:05,450 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:38:05,450 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:38:05,450 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:35 GMT)" executed successfully pullstatsredisflushworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'pullstatsredisflushworker.py', 'pid': '92'} pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stderr | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stderr | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stderr | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stderr | self.send(msg) pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stderr | self.connect() pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stderr | self.sock = self._create_connection( pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stderr | raise exceptions[0] pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stderr | sock.connect(sa) pullstatsredisflushworker stderr | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stderr | During handling of the above exception, another exception occurred: pullstatsredisflushworker stderr | Traceback (most recent call last): pullstatsredisflushworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stderr | push_to_gateway( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stderr | handler( pullstatsredisflushworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stderr | response = self._open(req, data) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stderr | result = func(*args) pullstatsredisflushworker stderr | ^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stderr | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stderr | raise URLError(err) pullstatsredisflushworker stderr | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:38:06,097 [93] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'queuecleanupworker.py', 'pid': '93'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:38:06,467 [103] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'servicekeyworker.py', 'pid': '103'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:38:07,086 [102] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityworker.py', 'pid': '102'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:38:07,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:38:07,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:17.122849+00:00 (in 9.999587 seconds) proxycacheblobworker stdout | 2025-11-04 09:38:07,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:17 GMT)" (scheduled at 2025-11-04 09:38:07.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:38:07,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:38:07,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 7, 123531), True, datetime.datetime(2025, 11, 4, 9, 38, 7, 123531), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:38:07,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:38:07,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:38:07,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:17 GMT)" executed successfully quotatotalworker stdout | 2025-11-04 09:38:07,512 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotatotalworker.py', 'pid': '95'} quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stdout | self.send(msg) quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stdout | self.connect() quotatotalworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stdout | self.sock = self._create_connection( quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stdout | raise exceptions[0] quotatotalworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stdout | sock.connect(sa) quotatotalworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stdout | During handling of the above exception, another exception occurred: quotatotalworker stdout | Traceback (most recent call last): quotatotalworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stdout | push_to_gateway( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stdout | handler( quotatotalworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stdout | response = self._open(req, data) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stdout | result = func(*args) quotatotalworker stdout | ^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stdout | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stdout | raise URLError(err) quotatotalworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:38:08,374 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:38:08,387 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:38:08,387 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:38:08,397 [66] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:38:08,435 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:38:08,435 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:21.434460+00:00 (in 12.998668 seconds) namespacegcworker stdout | 2025-11-04 09:38:08,435 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:43:08 GMT)" (scheduled at 2025-11-04 09:38:08.435432+00:00) namespacegcworker stdout | 2025-11-04 09:38:08,436 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 38, 8, 435998), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:38:08,437 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:38:08,437 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:35.435809+00:00 (in 26.998358 seconds) gcworker stdout | 2025-11-04 09:38:08,437 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:38 GMT)" (scheduled at 2025-11-04 09:38:08.437104+00:00) gcworker stdout | 2025-11-04 09:38:08,437 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) namespacegcworker stdout | 2025-11-04 09:38:08,448 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 38, 8, 435998), True, datetime.datetime(2025, 11, 4, 9, 38, 8, 435998), 0, 'namespacegc/%']) gcworker stdout | 2025-11-04 09:38:08,448 [72] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762249028448, None, 1, 0]) namespacegcworker stdout | 2025-11-04 09:38:08,451 [81] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 38, 8, 435998), True, datetime.datetime(2025, 11, 4, 9, 38, 8, 435998), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 38, 8, 435998), 'namespacegc/%']) gcworker stdout | 2025-11-04 09:38:08,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:38:08,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:38 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:38:08,454 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:38:08,455 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:43:08 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:38:08,561 [74] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '74'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,569 [298] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '298'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,575 [294] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '294'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,589 [297] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '297'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,617 [296] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '296'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,620 [295] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '295'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,642 [293] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '293'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,826 [299] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '299'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:38:08,826 [292] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'registry:application', 'pid': '292'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:38:08,935 [289] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '289'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:38:08,950 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '75'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:38:08,951 [290] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'secscan:application', 'pid': '290'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:38:09,133 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:38:09,134 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:57.131795+00:00 (in 47.997732 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:38:09,134 [79] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:09 GMT)" (scheduled at 2025-11-04 09:38:09.133645+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:38:09,134 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:38:09,146 [79] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:38:09,146 [79] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:38:09,146 [79] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:09 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:38:09,636 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'exportactionlogsworker.py', 'pid': '71'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:38:11,402 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestbackfillworker.py', 'pid': '78'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:38:11,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:38:11,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:15.535095+00:00 (in 4.004620 seconds) notificationworker stdout | 2025-11-04 09:38:11,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:21 GMT)" (scheduled at 2025-11-04 09:38:11.530064+00:00) notificationworker stdout | 2025-11-04 09:38:11,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:38:11,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 11, 530785), True, datetime.datetime(2025, 11, 4, 9, 38, 11, 530785), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:38:11,544 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:38:11,544 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:38:11,544 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:21 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:38:12,223 [94] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:38:12,223 [94] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:39:12.222828+00:00 (in 59.999560 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:38:12,223 [94] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:12 GMT)" (scheduled at 2025-11-04 09:38:12.222828+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:38:12,223 [94] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:38:12,234 [94] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:38:12,234 [94] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:12 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:38:12,869 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'buildlogsarchiver.py', 'pid': '67'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:38:14,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:38:14,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:29.351216+00:00 (in 14.999567 seconds) quotatotalworker stdout | 2025-11-04 09:38:14,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:38:29 GMT)" (scheduled at 2025-11-04 09:38:14.351216+00:00) quotatotalworker stdout | 2025-11-04 09:38:14,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:38:14,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:38:14,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:38:14,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:38:29 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:38:15,351 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'globalpromstats.py', 'pid': '73'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:38:15,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:38:15,480 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:17.479080+00:00 (in 1.998875 seconds) securityworker stdout | 2025-11-04 09:38:15,480 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:45 GMT)" (scheduled at 2025-11-04 09:38:15.479769+00:00) securityworker stdout | 2025-11-04 09:38:15,480 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:38:15,480 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:38:15,483 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:38:15,484 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:38:15,494 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:38:15,494 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:38:15,494 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:38:15,494 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:38:15,495 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 3]) securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:38:15,498 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:38:15,499 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 33, 15, 483923), 1, 3]) securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:38:15,502 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:38:15,503 [102] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 33, 15, 483923), 1, 3]) securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-3 by worker securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:38:15,506 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 3 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] Total range: 3-1 securityworker stderr | 2025-11-04 09:38:15 [102] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:38:15,506 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:45 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:38:15,535 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:38:15,535 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:21.530064+00:00 (in 5.994502 seconds) notificationworker stdout | 2025-11-04 09:38:15,535 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:15 GMT)" (scheduled at 2025-11-04 09:38:15.535095+00:00) notificationworker stdout | 2025-11-04 09:38:15,535 [88] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:38:15,535 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:15 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:38:15,696 [288] [DEBUG] [app] Starting request: urn:request:e28e1982-1114-4453-8b3e-405766fda9bb (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:38:15,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:15,700 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:38:15,700 [292] [DEBUG] [app] Starting request: urn:request:bf0608d8-05fc-42fb-be7d-dc62238a7781 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:38:15,701 [292] [DEBUG] [app] Ending request: urn:request:bf0608d8-05fc-42fb-be7d-dc62238a7781 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bf0608d8-05fc-42fb-be7d-dc62238a7781', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:38:15,701 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:38:15,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:15,702 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:15,704 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:38:15,705 [285] [DEBUG] [app] Starting request: urn:request:d6e445a3-2c66-40e2-8d1e-e71e00a18265 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:38:15,705 [285] [DEBUG] [app] Ending request: urn:request:d6e445a3-2c66-40e2-8d1e-e71e00a18265 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d6e445a3-2c66-40e2-8d1e-e71e00a18265', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:38:15,705 [285] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:38:15,705 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:15,706 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:38:15,706 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:38:15,706 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:38:15,713 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:38:15,713 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:38:15,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:38:15,726 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:38:15,729 [288] [DEBUG] [app] Ending request: urn:request:e28e1982-1114-4453-8b3e-405766fda9bb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e28e1982-1114-4453-8b3e-405766fda9bb', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:38:15,729 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:38:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.034) gunicorn-web stdout | 2025-11-04 09:38:15,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:38:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:38:15,783 [285] [DEBUG] [app] Starting request: urn:request:3bca58da-9689-4b31-a23f-a99bba1dc983 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:38:15,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:15,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:38:15,787 [292] [DEBUG] [app] Starting request: urn:request:82594075-35d3-46c5-8247-31ca9fb80cba (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:38:15,788 [292] [DEBUG] [app] Ending request: urn:request:82594075-35d3-46c5-8247-31ca9fb80cba (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:82594075-35d3-46c5-8247-31ca9fb80cba', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:38:15,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:15 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:38:15,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:15,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:15,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:38:15,792 [288] [DEBUG] [app] Starting request: urn:request:f0d060b7-bd18-4fd4-85fb-c7375ea2fca8 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:38:15,792 [288] [DEBUG] [app] Ending request: urn:request:f0d060b7-bd18-4fd4-85fb-c7375ea2fca8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f0d060b7-bd18-4fd4-85fb-c7375ea2fca8', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:38:15,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:15 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:15 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:38:15,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:15,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:38:15,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:38:15,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:38:15,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:38:15,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:38:15,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:38:15,813 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:38:15,816 [285] [DEBUG] [app] Ending request: urn:request:3bca58da-9689-4b31-a23f-a99bba1dc983 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3bca58da-9689-4b31-a23f-a99bba1dc983', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:38:15,816 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:38:15,816 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:38:15 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:38:15 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.033 47 0.034) proxycacheblobworker stdout | 2025-11-04 09:38:17,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:38:17,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:27.122849+00:00 (in 9.999641 seconds) proxycacheblobworker stdout | 2025-11-04 09:38:17,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:27 GMT)" (scheduled at 2025-11-04 09:38:17.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:38:17,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:38:17,123 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 17, 123428), True, datetime.datetime(2025, 11, 4, 9, 38, 17, 123428), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:38:17,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:38:17,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:38:17,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:27 GMT)" executed successfully securityworker stdout | 2025-11-04 09:38:17,479 [102] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:38:17,479 [102] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:45.479769+00:00 (in 28.000249 seconds) securityworker stdout | 2025-11-04 09:38:17,479 [102] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:47 GMT)" (scheduled at 2025-11-04 09:38:17.479080+00:00) securityworker stdout | 2025-11-04 09:38:17,479 [102] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:38:17,479 [102] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:38:17,481 [102] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:38:17,483 [102] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:38:17,494 [102] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:38:17,494 [102] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:47 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:38:18,287 [101] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'securityscanningnotificationworker.py', 'pid': '101'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:38:18,397 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:38:18,413 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:38:18,413 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:38:18,423 [66] [DEBUG] [data.database] Disconnecting from database. reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'reconciliationworker.py', 'pid': '96'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:38:18,968 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'proxycacheblobworker.py', 'pid': '91'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:38:20,272 [100] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositorygcworker.py', 'pid': '100'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:38:21,227 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:38:21,227 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:51.226294+00:00 (in 29.998947 seconds) autopruneworker stdout | 2025-11-04 09:38:21,227 [64] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:51 GMT)" (scheduled at 2025-11-04 09:38:21.226294+00:00) autopruneworker stdout | 2025-11-04 09:38:21,236 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762249041235, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:38:21,240 [64] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:38:21,240 [64] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:38:21,240 [64] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:51 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:38:21,434 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:38:21,434 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:32.435813+00:00 (in 11.000990 seconds) namespacegcworker stdout | 2025-11-04 09:38:21,434 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:21 GMT)" (scheduled at 2025-11-04 09:38:21.434460+00:00) namespacegcworker stdout | 2025-11-04 09:38:21,435 [81] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:38:21,435 [81] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 21, 435114), True, datetime.datetime(2025, 11, 4, 9, 38, 21, 435114), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:38:21,448 [81] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:38:21,448 [81] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:38:21,448 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:21 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:38:21,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:38:21,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:31.530064+00:00 (in 9.999624 seconds) notificationworker stdout | 2025-11-04 09:38:21,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:31 GMT)" (scheduled at 2025-11-04 09:38:21.530064+00:00) notificationworker stdout | 2025-11-04 09:38:21,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:38:21,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 21, 530778), True, datetime.datetime(2025, 11, 4, 9, 38, 21, 530778), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:38:21,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:38:21,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:38:21,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:31 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:38:22,548 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'autopruneworker.py', 'pid': '64'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:38:25,405 [88] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'notificationworker.py', 'pid': '88'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:38:27,123 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:38:27,123 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:37.122849+00:00 (in 9.999586 seconds) proxycacheblobworker stdout | 2025-11-04 09:38:27,123 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:37 GMT)" (scheduled at 2025-11-04 09:38:27.122849+00:00) proxycacheblobworker stdout | 2025-11-04 09:38:27,123 [91] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:38:27,124 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 27, 123527), True, datetime.datetime(2025, 11, 4, 9, 38, 27, 123527), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:38:27,136 [91] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:38:27,136 [91] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:38:27,136 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:37 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:38:27,223 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:38:27,223 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:57.222846+00:00 (in 29.999564 seconds) buildlogsarchiver stdout | 2025-11-04 09:38:27,223 [67] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:57 GMT)" (scheduled at 2025-11-04 09:38:27.222846+00:00) buildlogsarchiver stdout | 2025-11-04 09:38:27,223 [67] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 38, 27, 223541), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:38:27,236 [67] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."repository_id", "t1"."access_token_id", "t1"."resource_key", "t1"."job_config", "t1"."phase", "t1"."started", "t1"."display_name", "t1"."trigger_id", "t1"."pull_robot_id", "t1"."logs_archived", "t1"."queue_id" FROM "repositorybuild" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:38:27,239 [67] [DEBUG] [__main__] Archiving: a17b1ad2-20f8-4a4b-959f-611b223016a1 buildlogsarchiver stdout | 2025-11-04 09:38:27,240 [67] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:38:27,245 [67] [ERROR] [workers.worker] Operation raised exception buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/workers/worker.py", line 126, in _operation_func buildlogsarchiver stdout | return operation_func() buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/quay-registry/workers/buildlogsarchiver/buildlogsarchiver.py", line 55, in _archive_redis_buildlogs buildlogsarchiver stdout | log_archive.store_file( buildlogsarchiver stdout | File "/quay-registry/data/userfiles.py", line 112, in store_file buildlogsarchiver stdout | self._storage.stream_write( buildlogsarchiver stdout | File "/quay-registry/storage/distributedstorage.py", line 23, in wrapper buildlogsarchiver stdout | storage = self._storages[random.sample(locations, 1)[0]] buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/random.py", line 413, in sample buildlogsarchiver stdout | raise TypeError("Population must be a sequence. " buildlogsarchiver stdout | TypeError: Population must be a sequence. For dicts or sets, use sorted(d). buildlogsarchiver stdout | 2025-11-04 09:38:27,246 [67] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:38:57 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:38:27,627 [94] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'quotaregistrysizeworker.py', 'pid': '94'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:38:28,339 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'logrotateworker.py', 'pid': '77'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:38:28,423 [66] [DEBUG] [buildman.manager.ephemeral] Checking for more work from the build queue builder stdout | 2025-11-04 09:38:28,437 [66] [DEBUG] [buildman.manager.ephemeral] No additional work found. Going to sleep for 10 seconds builder stdout | 2025-11-04 09:38:28,437 [66] [DEBUG] [buildman.manager.ephemeral] Writing queue metrics builder stdout | 2025-11-04 09:38:28,447 [66] [DEBUG] [data.database] Disconnecting from database. storagereplication stdout | 2025-11-04 09:38:28,991 [104] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'storagereplication.py', 'pid': '104'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: quotatotalworker stdout | 2025-11-04 09:38:29,351 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotatotalworker stdout | 2025-11-04 09:38:29,351 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:44.351216+00:00 (in 14.999528 seconds) quotatotalworker stdout | 2025-11-04 09:38:29,351 [95] [INFO] [apscheduler.executors.default] Running job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:38:44 GMT)" (scheduled at 2025-11-04 09:38:29.351216+00:00) quotatotalworker stdout | 2025-11-04 09:38:29,351 [95] [INFO] [__main__] Quota backfill worker started, searching for namespaces to calculate size quotatotalworker stdout | 2025-11-04 09:38:29,352 [95] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ((NOT EXISTS(SELECT "t2"."id" FROM "quotanamespacesize" AS "t2" WHERE (("t2"."namespace_user_id" = "t1"."id") AND ("t2"."backfill_start_ms" IS NOT %s))) AND ("t1"."enabled" = %s)) AND ("t1"."robot" = %s)) LIMIT %s', [None, True, False, 100]) quotatotalworker stdout | 2025-11-04 09:38:29,364 [95] [DEBUG] [data.database] Disconnecting from database. quotatotalworker stdout | 2025-11-04 09:38:29,364 [95] [INFO] [apscheduler.executors.default] Job "QuotaTotalWorker.backfill (trigger: interval[0:00:15], next run at: 2025-11-04 09:38:44 GMT)" executed successfully teamsyncworker stdout | 2025-11-04 09:38:29,373 [105] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'teamsyncworker.py', 'pid': '105'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:29,805 [76] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '76'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:30,100 [286] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '286'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:30,284 [285] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '285'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:30,293 [287] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '287'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:30,390 [288] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'web:application', 'pid': '288'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:38:30,696 [288] [DEBUG] [app] Starting request: urn:request:f7e63636-15c4-4422-a1d2-09bb8ba93a01 (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:38:30,697 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:30,699 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:38:30,700 [299] [DEBUG] [app] Starting request: urn:request:073bb4ce-1fef-4bc7-bfb4-44befd54639b (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:38:30,700 [299] [DEBUG] [app] Ending request: urn:request:073bb4ce-1fef-4bc7-bfb4-44befd54639b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:073bb4ce-1fef-4bc7-bfb4-44befd54639b', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:38:30,700 [299] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:38:30,701 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:30,701 [288] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:30,703 [288] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:38:30,704 [287] [DEBUG] [app] Starting request: urn:request:a49a3085-d909-46eb-b6b4-e9f358540c54 (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:38:30,704 [287] [DEBUG] [app] Ending request: urn:request:a49a3085-d909-46eb-b6b4-e9f358540c54 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a49a3085-d909-46eb-b6b4-e9f358540c54', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:38:30,705 [287] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:38:30,705 [288] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:30,705 [288] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:38:30,705 [288] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:38:30,705 [288] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:38:30,713 [288] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:38:30,713 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:38:30,723 [288] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:38:30,726 [288] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:38:30,729 [288] [DEBUG] [app] Ending request: urn:request:f7e63636-15c4-4422-a1d2-09bb8ba93a01 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f7e63636-15c4-4422-a1d2-09bb8ba93a01', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:38:30,729 [288] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:38:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.035 47 0.035) gunicorn-web stdout | 2025-11-04 09:38:30,730 [288] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:38:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:38:30,783 [285] [DEBUG] [app] Starting request: urn:request:fe35b233-8b87-4470-a547-ca657937282a (/health/instance) {'X-Forwarded-For': '10.131.0.2'} gunicorn-web stdout | 2025-11-04 09:38:30,784 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:30,787 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-registry stdout | 2025-11-04 09:38:30,788 [292] [DEBUG] [app] Starting request: urn:request:8b668bae-a26b-4f7d-a42f-517b2c0951e5 (/v1/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-registry stdout | 2025-11-04 09:38:30,788 [292] [DEBUG] [app] Ending request: urn:request:8b668bae-a26b-4f7d-a42f-517b2c0951e5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8b668bae-a26b-4f7d-a42f-517b2c0951e5', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '76b27b45', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:38:30,788 [292] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:30 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:38:30,788 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:30,789 [285] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:38:30,791 [285] [WARNING] [py.warnings] /opt/app-root/lib64/python3.12/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings gunicorn-web stdout | warnings.warn( gunicorn-web stdout | 2025-11-04 09:38:30,792 [288] [DEBUG] [app] Starting request: urn:request:b8b2cf5b-1f59-4d72-9d43-17ba47f5b65f (/_internal_ping) {'X-Forwarded-For': '::1'} gunicorn-web stdout | 2025-11-04 09:38:30,792 [288] [DEBUG] [app] Ending request: urn:request:b8b2cf5b-1f59-4d72-9d43-17ba47f5b65f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b8b2cf5b-1f59-4d72-9d43-17ba47f5b65f', 'remote_addr': '::1', 'http_method': 'GET', 'original_url': 'https://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | ::1 (-) - - [04/Nov/2025:09:38:30 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:38:30,792 [288] [INFO] [gunicorn.access] ::1 - - [04/Nov/2025:09:38:30 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:38:30,792 [285] [DEBUG] [urllib3.connectionpool] https://localhost:8443 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:38:30,793 [285] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:38:30,793 [285] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:38:30,793 [285] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:38:30,800 [285] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:38:30,800 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:38:30,810 [285] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:38:30,814 [285] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:38:30,817 [285] [DEBUG] [app] Ending request: urn:request:fe35b233-8b87-4470-a547-ca657937282a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fe35b233-8b87-4470-a547-ca657937282a', 'remote_addr': '10.131.0.2', 'http_method': 'GET', 'original_url': 'https://10.131.0.41/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '12ace70d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:38:30,817 [285] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:38:30,817 [285] [INFO] [gunicorn.access] 10.131.0.2 - - [04/Nov/2025:09:38:30 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.0.2 (-) - - [04/Nov/2025:09:38:30 +0000] "GET /health/instance HTTP/2.0" 200 152 "-" "kube-probe/1.32" (0.034 47 0.034) notificationworker stdout | 2025-11-04 09:38:31,530 [88] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:38:31,530 [88] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:41.530064+00:00 (in 9.999544 seconds) notificationworker stdout | 2025-11-04 09:38:31,530 [88] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:41 GMT)" (scheduled at 2025-11-04 09:38:31.530064+00:00) notificationworker stdout | 2025-11-04 09:38:31,530 [88] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:38:31,531 [88] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 31, 530780), True, datetime.datetime(2025, 11, 4, 9, 38, 31, 530780), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:38:31,543 [88] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:38:31,543 [88] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:38:31,543 [88] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:38:41 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:38:31,632 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '79'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:38:32,429 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'gcworker.py', 'pid': '72'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:38:32,436 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:38:32,436 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:39:21.434460+00:00 (in 48.998255 seconds) namespacegcworker stdout | 2025-11-04 09:38:32,436 [81] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:32 GMT)" (scheduled at 2025-11-04 09:38:32.435813+00:00) namespacegcworker stdout | 2025-11-04 09:38:32,436 [81] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:38:32,436 [81] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:32 GMT)" executed successfully repositoryactioncounter stdout | 2025-11-04 09:38:32,625 [98] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'repositoryactioncounter.py', 'pid': '98'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:38:32,630 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'blobuploadcleanupworker.py', 'pid': '65'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:38:33,623 [71] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:38:33,623 [71] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:48.625192+00:00 (in 15.001822 seconds) exportactionlogsworker stdout | 2025-11-04 09:38:33,623 [71] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:33 GMT)" (scheduled at 2025-11-04 09:38:33.622985+00:00) exportactionlogsworker stdout | 2025-11-04 09:38:33,623 [71] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:38:33,624 [71] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 38, 33, 623702), True, datetime.datetime(2025, 11, 4, 9, 38, 33, 623702), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:38:33,636 [71] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:38:33,636 [71] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:38:33,636 [71] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:39:33 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:38:33,911 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '70'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:38:33,905 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'chunkcleanupworker.py', 'pid': '68'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:38:34,007 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'namespacegcworker.py', 'pid': '81'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:38:34,794 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-697f89d66-k62z8', 'process_name': 'builder.py', 'pid': '66'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:38:35,436 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:38:35,436 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:38:38.437104+00:00 (in 3.000521 seconds) gcworker stdout | 2025-11-04 09:38:35,436 [72] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:39:05 GMT)" (scheduled at 2025-11-04 09:38:35.435809+00:00) gcworker stdout | 2025-11-04 09:38:35,437 [72] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761039515436, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:38:35,451 [72] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:38:35,451 [72] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:38:35,452 [72] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:39:05 GMT)" executed successfully